2024-12-03 06:12:56,116 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 06:12:56,127 main DEBUG Took 0.008792 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 06:12:56,127 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 06:12:56,127 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 06:12:56,128 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 06:12:56,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,135 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 06:12:56,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,148 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,149 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,151 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,152 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,153 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,154 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,155 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,155 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:12:56,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,156 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 06:12:56,158 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:12:56,159 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 06:12:56,160 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 06:12:56,161 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 06:12:56,162 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 06:12:56,162 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 06:12:56,170 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 06:12:56,172 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 06:12:56,174 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 06:12:56,174 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 06:12:56,174 main DEBUG createAppenders(={Console}) 2024-12-03 06:12:56,175 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 06:12:56,175 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 06:12:56,176 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 06:12:56,176 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 06:12:56,176 main DEBUG OutputStream closed 2024-12-03 06:12:56,176 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 06:12:56,177 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 06:12:56,177 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 06:12:56,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 06:12:56,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 06:12:56,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 06:12:56,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 06:12:56,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 06:12:56,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 06:12:56,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 06:12:56,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 06:12:56,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 06:12:56,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 06:12:56,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 06:12:56,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 06:12:56,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 06:12:56,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 06:12:56,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 06:12:56,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 06:12:56,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 06:12:56,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 06:12:56,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 06:12:56,251 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 06:12:56,251 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 06:12:56,252 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T06:12:56,479 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998 2024-12-03 06:12:56,483 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 06:12:56,483 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T06:12:56,491 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T06:12:56,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T06:12:56,528 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38, deleteOnExit=true 2024-12-03T06:12:56,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T06:12:56,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/test.cache.data in system properties and HBase conf 2024-12-03T06:12:56,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T06:12:56,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.log.dir in system properties and HBase conf 2024-12-03T06:12:56,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T06:12:56,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T06:12:56,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T06:12:56,614 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T06:12:56,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T06:12:56,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T06:12:56,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T06:12:56,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T06:12:56,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T06:12:56,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T06:12:56,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T06:12:56,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T06:12:56,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T06:12:56,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T06:12:56,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/nfs.dump.dir in system properties and HBase conf 2024-12-03T06:12:56,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/java.io.tmpdir in system properties and HBase conf 2024-12-03T06:12:56,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T06:12:56,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T06:12:56,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T06:12:57,691 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T06:12:57,767 INFO [Time-limited test {}] log.Log(170): Logging initialized @2264ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T06:12:57,837 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:12:57,897 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:12:57,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:12:57,920 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:12:57,921 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T06:12:57,936 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:12:57,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:12:57,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:12:58,109 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/java.io.tmpdir/jetty-localhost-38829-hadoop-hdfs-3_4_1-tests_jar-_-any-16426336743201569228/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T06:12:58,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38829} 2024-12-03T06:12:58,116 INFO [Time-limited test {}] server.Server(415): Started @2612ms 2024-12-03T06:12:58,631 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:12:58,639 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:12:58,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:12:58,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:12:58,640 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T06:12:58,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:12:58,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:12:58,739 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/java.io.tmpdir/jetty-localhost-39997-hadoop-hdfs-3_4_1-tests_jar-_-any-11817152089162405208/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:12:58,740 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39997} 2024-12-03T06:12:58,740 INFO [Time-limited test {}] server.Server(415): Started @3237ms 2024-12-03T06:12:58,786 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T06:12:59,663 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data2/current/BP-1574928969-172.17.0.2-1733206377249/current, will proceed with Du for space computation calculation, 2024-12-03T06:12:59,663 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data1/current/BP-1574928969-172.17.0.2-1733206377249/current, will proceed with Du for space computation calculation, 2024-12-03T06:12:59,691 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T06:12:59,733 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc5b6fee131c4203 with lease ID 0xbb84b5f2d70b5923: Processing first storage report for DS-585999c3-9b04-4588-993b-737eb84028b6 from datanode DatanodeRegistration(127.0.0.1:45259, datanodeUuid=5db34709-8f31-43be-8f27-fa9b2176541b, infoPort=39589, infoSecurePort=0, ipcPort=37361, storageInfo=lv=-57;cid=testClusterID;nsid=2115516907;c=1733206377249) 2024-12-03T06:12:59,734 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc5b6fee131c4203 with lease ID 0xbb84b5f2d70b5923: from storage DS-585999c3-9b04-4588-993b-737eb84028b6 node DatanodeRegistration(127.0.0.1:45259, datanodeUuid=5db34709-8f31-43be-8f27-fa9b2176541b, infoPort=39589, infoSecurePort=0, ipcPort=37361, storageInfo=lv=-57;cid=testClusterID;nsid=2115516907;c=1733206377249), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T06:12:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc5b6fee131c4203 with lease ID 0xbb84b5f2d70b5923: Processing first storage report for DS-3bba9cb8-f890-4010-8aef-54e1bdeb8ccd from datanode DatanodeRegistration(127.0.0.1:45259, datanodeUuid=5db34709-8f31-43be-8f27-fa9b2176541b, infoPort=39589, infoSecurePort=0, ipcPort=37361, storageInfo=lv=-57;cid=testClusterID;nsid=2115516907;c=1733206377249) 2024-12-03T06:12:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc5b6fee131c4203 with lease ID 0xbb84b5f2d70b5923: from storage DS-3bba9cb8-f890-4010-8aef-54e1bdeb8ccd node DatanodeRegistration(127.0.0.1:45259, datanodeUuid=5db34709-8f31-43be-8f27-fa9b2176541b, infoPort=39589, infoSecurePort=0, ipcPort=37361, storageInfo=lv=-57;cid=testClusterID;nsid=2115516907;c=1733206377249), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T06:12:59,772 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998 2024-12-03T06:12:59,852 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/zookeeper_0, clientPort=53118, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T06:12:59,861 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53118 2024-12-03T06:12:59,870 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:12:59,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:00,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741825_1001 (size=7) 2024-12-03T06:13:00,529 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c with version=8 2024-12-03T06:13:00,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/hbase-staging 2024-12-03T06:13:00,635 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T06:13:00,858 INFO [Time-limited test {}] client.ConnectionUtils(128): master/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:13:00,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:00,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:00,871 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:13:00,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:00,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:13:00,987 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T06:13:01,050 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T06:13:01,059 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T06:13:01,063 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:13:01,087 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3066 (auto-detected) 2024-12-03T06:13:01,088 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T06:13:01,106 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42621 2024-12-03T06:13:01,127 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42621 connecting to ZooKeeper ensemble=127.0.0.1:53118 2024-12-03T06:13:01,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426210x0, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:13:01,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42621-0x1009ea88bf20000 connected 2024-12-03T06:13:01,310 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:01,314 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:01,326 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:13:01,330 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c, hbase.cluster.distributed=false 2024-12-03T06:13:01,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:13:01,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42621 2024-12-03T06:13:01,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42621 2024-12-03T06:13:01,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42621 2024-12-03T06:13:01,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42621 2024-12-03T06:13:01,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42621 2024-12-03T06:13:01,459 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:13:01,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:01,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:01,462 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:13:01,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:13:01,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:13:01,465 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T06:13:01,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:13:01,469 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35255 2024-12-03T06:13:01,471 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35255 connecting to ZooKeeper ensemble=127.0.0.1:53118 2024-12-03T06:13:01,472 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:01,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:01,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352550x0, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:13:01,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352550x0, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:13:01,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35255-0x1009ea88bf20001 connected 2024-12-03T06:13:01,495 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T06:13:01,503 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T06:13:01,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T06:13:01,513 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:13:01,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35255 2024-12-03T06:13:01,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35255 2024-12-03T06:13:01,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35255 2024-12-03T06:13:01,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35255 2024-12-03T06:13:01,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35255 2024-12-03T06:13:01,538 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;122cddc95ec8:42621 2024-12-03T06:13:01,539 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/122cddc95ec8,42621,1733206380709 2024-12-03T06:13:01,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:13:01,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:13:01,553 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/122cddc95ec8,42621,1733206380709 2024-12-03T06:13:01,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:01,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T06:13:01,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:01,582 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T06:13:01,583 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/122cddc95ec8,42621,1733206380709 from backup master directory 2024-12-03T06:13:01,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:13:01,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/122cddc95ec8,42621,1733206380709 2024-12-03T06:13:01,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:13:01,593 WARN [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:13:01,594 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=122cddc95ec8,42621,1733206380709 2024-12-03T06:13:01,596 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T06:13:01,597 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T06:13:01,652 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/hbase.id] with ID: 8ff2d4dc-7837-43e0-8545-df22b9939adf 2024-12-03T06:13:01,652 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/.tmp/hbase.id 2024-12-03T06:13:01,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741826_1002 (size=42) 2024-12-03T06:13:02,065 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/.tmp/hbase.id]:[hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/hbase.id] 2024-12-03T06:13:02,115 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:13:02,119 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T06:13:02,137 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T06:13:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:02,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741827_1003 (size=196) 2024-12-03T06:13:02,618 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:13:02,621 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T06:13:02,635 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:02,639 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:13:02,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741828_1004 (size=1189) 2024-12-03T06:13:03,097 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store 2024-12-03T06:13:03,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741829_1005 (size=34) 2024-12-03T06:13:03,527 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T06:13:03,530 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:03,531 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T06:13:03,531 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:13:03,531 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:13:03,532 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T06:13:03,533 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:13:03,533 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:13:03,534 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733206383531Disabling compacts and flushes for region at 1733206383531Disabling writes for close at 1733206383532 (+1 ms)Writing region close event to WAL at 1733206383533 (+1 ms)Closed at 1733206383533 2024-12-03T06:13:03,536 WARN [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/.initializing 2024-12-03T06:13:03,536 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/WALs/122cddc95ec8,42621,1733206380709 2024-12-03T06:13:03,544 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:13:03,558 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C42621%2C1733206380709, suffix=, logDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/WALs/122cddc95ec8,42621,1733206380709, archiveDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/oldWALs, maxLogs=10 2024-12-03T06:13:03,578 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/WALs/122cddc95ec8,42621,1733206380709/122cddc95ec8%2C42621%2C1733206380709.1733206383562, exclude list is [], retry=0 2024-12-03T06:13:03,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45259,DS-585999c3-9b04-4588-993b-737eb84028b6,DISK] 2024-12-03T06:13:03,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T06:13:03,627 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/WALs/122cddc95ec8,42621,1733206380709/122cddc95ec8%2C42621%2C1733206380709.1733206383562 2024-12-03T06:13:03,628 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39589:39589)] 2024-12-03T06:13:03,629 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:13:03,629 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:03,632 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,633 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T06:13:03,701 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:03,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:03,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T06:13:03,707 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:03,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:03,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T06:13:03,712 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:03,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:03,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T06:13:03,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:03,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:03,718 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,722 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,723 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,728 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,729 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,733 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T06:13:03,736 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:13:03,741 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:13:03,742 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69100422, jitterRate=0.02967652678489685}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T06:13:03,747 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733206383645Initializing all the Stores at 1733206383648 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206383648Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206383649 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206383649Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206383649Cleaning up temporary data from old regions at 1733206383729 (+80 ms)Region opened successfully at 1733206383747 (+18 ms) 2024-12-03T06:13:03,749 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T06:13:03,786 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29efa2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:13:03,818 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T06:13:03,829 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T06:13:03,829 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T06:13:03,832 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:13:03,834 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T06:13:03,840 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T06:13:03,841 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T06:13:03,868 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T06:13:03,881 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T06:13:03,914 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T06:13:03,918 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T06:13:03,920 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T06:13:03,934 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T06:13:03,937 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T06:13:03,944 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T06:13:03,951 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T06:13:03,953 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T06:13:03,964 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T06:13:03,991 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T06:13:04,001 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T06:13:04,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:13:04,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:13:04,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,019 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=122cddc95ec8,42621,1733206380709, sessionid=0x1009ea88bf20000, setting cluster-up flag (Was=false) 2024-12-03T06:13:04,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,076 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T06:13:04,078 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=122cddc95ec8,42621,1733206380709 2024-12-03T06:13:04,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:04,134 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T06:13:04,138 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=122cddc95ec8,42621,1733206380709 2024-12-03T06:13:04,147 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T06:13:04,232 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(746): ClusterId : 8ff2d4dc-7837-43e0-8545-df22b9939adf 2024-12-03T06:13:04,236 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T06:13:04,251 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T06:13:04,251 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T06:13:04,255 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:13:04,261 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T06:13:04,262 DEBUG [RS:0;122cddc95ec8:35255 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338d9b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:13:04,270 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T06:13:04,278 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;122cddc95ec8:35255 2024-12-03T06:13:04,279 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T06:13:04,282 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T06:13:04,283 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T06:13:04,283 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T06:13:04,287 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,42621,1733206380709 with port=35255, startcode=1733206381424 2024-12-03T06:13:04,287 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 122cddc95ec8,42621,1733206380709 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T06:13:04,298 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:13:04,299 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:13:04,300 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:13:04,301 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:13:04,302 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/122cddc95ec8:0, corePoolSize=10, maxPoolSize=10 2024-12-03T06:13:04,305 DEBUG [RS:0;122cddc95ec8:35255 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T06:13:04,306 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,306 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:13:04,306 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,311 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206414311 2024-12-03T06:13:04,313 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T06:13:04,315 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T06:13:04,315 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:13:04,316 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T06:13:04,319 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T06:13:04,319 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T06:13:04,320 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T06:13:04,320 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T06:13:04,323 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:04,323 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T06:13:04,322 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,326 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T06:13:04,328 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T06:13:04,328 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T06:13:04,336 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T06:13:04,337 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T06:13:04,350 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206384341,5,FailOnTimeoutGroup] 2024-12-03T06:13:04,351 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206384350,5,FailOnTimeoutGroup] 2024-12-03T06:13:04,352 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,352 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T06:13:04,354 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51737, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T06:13:04,355 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741831_1007 (size=1321) 2024-12-03T06:13:04,362 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42621 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,364 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42621 {}] master.ServerManager(517): Registering regionserver=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,381 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:13:04,382 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44351 2024-12-03T06:13:04,382 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T06:13:04,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T06:13:04,393 DEBUG [RS:0;122cddc95ec8:35255 {}] zookeeper.ZKUtil(111): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,394 WARN [RS:0;122cddc95ec8:35255 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:13:04,394 INFO [RS:0;122cddc95ec8:35255 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:13:04,394 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,409 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [122cddc95ec8,35255,1733206381424] 2024-12-03T06:13:04,430 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T06:13:04,446 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T06:13:04,453 INFO [RS:0;122cddc95ec8:35255 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T06:13:04,454 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,455 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T06:13:04,462 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T06:13:04,464 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,464 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,465 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,465 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,465 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,465 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,465 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,466 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:13:04,467 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:13:04,467 DEBUG [RS:0;122cddc95ec8:35255 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:13:04,470 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,471 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,471 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,471 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,472 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,472 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35255,1733206381424-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:13:04,491 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T06:13:04,493 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35255,1733206381424-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,494 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,494 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.Replication(171): 122cddc95ec8,35255,1733206381424 started 2024-12-03T06:13:04,510 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:04,511 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1482): Serving as 122cddc95ec8,35255,1733206381424, RpcServer on 122cddc95ec8/172.17.0.2:35255, sessionid=0x1009ea88bf20001 2024-12-03T06:13:04,512 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T06:13:04,512 DEBUG [RS:0;122cddc95ec8:35255 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,512 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,35255,1733206381424' 2024-12-03T06:13:04,512 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T06:13:04,513 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T06:13:04,514 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T06:13:04,514 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T06:13:04,514 DEBUG [RS:0;122cddc95ec8:35255 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 122cddc95ec8,35255,1733206381424 2024-12-03T06:13:04,515 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,35255,1733206381424' 2024-12-03T06:13:04,515 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T06:13:04,515 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T06:13:04,516 DEBUG [RS:0;122cddc95ec8:35255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T06:13:04,516 INFO [RS:0;122cddc95ec8:35255 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T06:13:04,516 INFO [RS:0;122cddc95ec8:35255 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T06:13:04,626 INFO [RS:0;122cddc95ec8:35255 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:13:04,631 INFO [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C35255%2C1733206381424, suffix=, logDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424, archiveDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/oldWALs, maxLogs=32 2024-12-03T06:13:04,647 DEBUG [RS:0;122cddc95ec8:35255 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424/122cddc95ec8%2C35255%2C1733206381424.1733206384634, exclude list is [], retry=0 2024-12-03T06:13:04,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45259,DS-585999c3-9b04-4588-993b-737eb84028b6,DISK] 2024-12-03T06:13:04,655 INFO [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424/122cddc95ec8%2C35255%2C1733206381424.1733206384634 2024-12-03T06:13:04,656 DEBUG [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39589:39589)] 2024-12-03T06:13:04,760 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T06:13:04,761 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:13:04,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741833_1009 (size=32) 2024-12-03T06:13:04,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:04,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T06:13:04,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T06:13:04,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:04,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:04,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T06:13:04,789 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T06:13:04,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:04,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:04,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T06:13:04,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T06:13:04,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:04,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:04,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T06:13:04,799 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T06:13:04,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:04,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:04,801 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T06:13:04,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740 2024-12-03T06:13:04,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740 2024-12-03T06:13:04,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T06:13:04,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T06:13:04,807 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:13:04,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T06:13:04,814 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:13:04,815 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62981985, jitterRate=-0.06149528920650482}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:13:04,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733206384779Initializing all the Stores at 1733206384781 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206384781Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206384781Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206384781Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206384781Cleaning up temporary data from old regions at 1733206384806 (+25 ms)Region opened successfully at 1733206384817 (+11 ms) 2024-12-03T06:13:04,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T06:13:04,817 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T06:13:04,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T06:13:04,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T06:13:04,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T06:13:04,819 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T06:13:04,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733206384817Disabling compacts and flushes for region at 1733206384817Disabling writes for close at 1733206384817Writing region close event to WAL at 1733206384819 (+2 ms)Closed at 1733206384819 2024-12-03T06:13:04,823 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:13:04,824 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T06:13:04,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T06:13:04,839 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:13:04,841 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T06:13:04,994 DEBUG [122cddc95ec8:42621 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T06:13:05,005 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:05,012 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,35255,1733206381424, state=OPENING 2024-12-03T06:13:05,047 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T06:13:05,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:05,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:13:05,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:13:05,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:13:05,058 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:13:05,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:05,245 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T06:13:05,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T06:13:05,258 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T06:13:05,259 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:13:05,259 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T06:13:05,263 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C35255%2C1733206381424.meta, suffix=.meta, logDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424, archiveDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/oldWALs, maxLogs=32 2024-12-03T06:13:05,275 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424/122cddc95ec8%2C35255%2C1733206381424.meta.1733206385264.meta, exclude list is [], retry=0 2024-12-03T06:13:05,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45259,DS-585999c3-9b04-4588-993b-737eb84028b6,DISK] 2024-12-03T06:13:05,282 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/WALs/122cddc95ec8,35255,1733206381424/122cddc95ec8%2C35255%2C1733206381424.meta.1733206385264.meta 2024-12-03T06:13:05,282 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39589:39589)] 2024-12-03T06:13:05,283 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:13:05,284 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T06:13:05,288 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T06:13:05,292 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T06:13:05,297 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T06:13:05,297 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:05,298 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T06:13:05,298 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T06:13:05,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T06:13:05,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T06:13:05,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:05,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:05,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T06:13:05,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T06:13:05,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:05,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:05,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T06:13:05,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T06:13:05,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:05,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:05,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T06:13:05,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T06:13:05,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:05,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:13:05,312 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T06:13:05,314 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740 2024-12-03T06:13:05,317 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740 2024-12-03T06:13:05,320 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T06:13:05,320 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T06:13:05,321 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:13:05,324 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T06:13:05,325 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60362561, jitterRate=-0.10052774846553802}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:13:05,326 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T06:13:05,327 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733206385298Writing region info on filesystem at 1733206385299 (+1 ms)Initializing all the Stores at 1733206385301 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206385301Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206385301Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206385301Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206385301Cleaning up temporary data from old regions at 1733206385320 (+19 ms)Running coprocessor post-open hooks at 1733206385326 (+6 ms)Region opened successfully at 1733206385327 (+1 ms) 2024-12-03T06:13:05,333 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733206385237 2024-12-03T06:13:05,343 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T06:13:05,344 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T06:13:05,346 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:05,349 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,35255,1733206381424, state=OPEN 2024-12-03T06:13:05,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:13:05,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:13:05,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:13:05,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:13:05,410 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:05,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T06:13:05,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,35255,1733206381424 in 350 msec 2024-12-03T06:13:05,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T06:13:05,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 588 msec 2024-12-03T06:13:05,425 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:13:05,425 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T06:13:05,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:05,445 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:05,463 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35493, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:05,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2910 sec 2024-12-03T06:13:05,487 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733206385487, completionTime=-1 2024-12-03T06:13:05,490 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T06:13:05,491 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:13:05,517 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T06:13:05,517 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206445517 2024-12-03T06:13:05,517 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733206505517 2024-12-03T06:13:05,518 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-03T06:13:05,520 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,520 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,520 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,522 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-122cddc95ec8:42621, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,522 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,523 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,528 DEBUG [master/122cddc95ec8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T06:13:05,547 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.953sec 2024-12-03T06:13:05,548 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T06:13:05,549 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T06:13:05,551 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T06:13:05,551 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T06:13:05,551 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T06:13:05,552 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:13:05,553 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T06:13:05,562 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T06:13:05,563 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T06:13:05,564 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,42621,1733206380709-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:13:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:05,649 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T06:13:05,649 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T06:13:05,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:05,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:05,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:05,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:05,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:05,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:05,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:05,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=122cddc95ec8,42621,1733206380709 2024-12-03T06:13:05,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=280, ProcessCount=11, AvailableMemoryMB=12419 2024-12-03T06:13:05,730 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:13:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:13:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-12-03T06:13:05,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:13:05,742 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:13:05,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:13:05,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:13:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:05,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:13:05,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T06:13:05,762 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:05,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:13:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:05,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741835_1011 (size=1061) 2024-12-03T06:13:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:06,187 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:13:06,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741836_1012 (size=53) 2024-12-03T06:13:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8ec9e85614363b6e4e059ef6575a1df6, disabling compactions & flushes 2024-12-03T06:13:06,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. after waiting 0 ms 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ec9e85614363b6e4e059ef6575a1df6: Waiting for close lock at 1733206386599Disabling compacts and flushes for region at 1733206386599Disabling writes for close at 1733206386599Writing region close event to WAL at 1733206386599Closed at 1733206386599 2024-12-03T06:13:06,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:13:06,607 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206386602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206386602"}]},"ts":"1733206386602"} 2024-12-03T06:13:06,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:13:06,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:13:06,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206386614"}]},"ts":"1733206386614"} 2024-12-03T06:13:06,620 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:13:06,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, ASSIGN}] 2024-12-03T06:13:06,625 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, ASSIGN 2024-12-03T06:13:06,628 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:13:06,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ec9e85614363b6e4e059ef6575a1df6, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:06,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, ASSIGN because future has completed 2024-12-03T06:13:06,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:06,951 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,952 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:13:06,952 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,952 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:06,953 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,953 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,955 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,960 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:06,961 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ec9e85614363b6e4e059ef6575a1df6 columnFamilyName A 2024-12-03T06:13:06,961 DEBUG [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:06,963 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(327): Store=8ec9e85614363b6e4e059ef6575a1df6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:06,963 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,966 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:06,967 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ec9e85614363b6e4e059ef6575a1df6 columnFamilyName B 2024-12-03T06:13:06,967 DEBUG [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:06,968 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(327): Store=8ec9e85614363b6e4e059ef6575a1df6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:06,968 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,973 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:06,974 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ec9e85614363b6e4e059ef6575a1df6 columnFamilyName C 2024-12-03T06:13:06,974 DEBUG [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:06,975 INFO [StoreOpener-8ec9e85614363b6e4e059ef6575a1df6-1 {}] regionserver.HStore(327): Store=8ec9e85614363b6e4e059ef6575a1df6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:06,976 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,977 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,978 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,979 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,982 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,982 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,984 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:13:06,986 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,990 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:13:06,991 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8ec9e85614363b6e4e059ef6575a1df6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62124225, jitterRate=-0.07427690923213959}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:13:06,991 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:06,992 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8ec9e85614363b6e4e059ef6575a1df6: Running coprocessor pre-open hook at 1733206386953Writing region info on filesystem at 1733206386953Initializing all the Stores at 1733206386955 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206386955Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206386955Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206386955Cleaning up temporary data from old regions at 1733206386983 (+28 ms)Running coprocessor post-open hooks at 1733206386991 (+8 ms)Region opened successfully at 1733206386992 (+1 ms) 2024-12-03T06:13:06,994 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., pid=6, masterSystemTime=1733206386944 2024-12-03T06:13:06,998 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:06,998 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:07,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ec9e85614363b6e4e059ef6575a1df6, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:07,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:13:07,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T06:13:07,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 in 219 msec 2024-12-03T06:13:07,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T06:13:07,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, ASSIGN in 389 msec 2024-12-03T06:13:07,017 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:13:07,017 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206387017"}]},"ts":"1733206387017"} 2024-12-03T06:13:07,021 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:13:07,023 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:13:07,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-12-03T06:13:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:13:07,931 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:07,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:07,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:07,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bff7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:07,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2538dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:07,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:07,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:13:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:13:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@481dbf66 2024-12-03T06:13:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:13:07,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:13:07,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:07,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:07,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:07,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:07,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e082570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:07,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:07,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:07,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:07,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:07,987 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:07,992 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:07,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:07,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:08,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:08,004 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,007 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:08,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:08,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df9bce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,015 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:08,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:08,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b8551d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517e68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:08,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:08,026 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1b5617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:08,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:08,032 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373a2bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:08,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9632d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:08,039 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,039 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,039 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,041 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,041 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,041 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,042 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,045 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:08,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T06:13:08,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:08,074 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:08,077 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:08,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:08,079 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T06:13:08,080 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:08,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:08,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:13:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:08,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:13:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:08,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T06:13:08,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T06:13:08,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206448292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206448294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206448301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206448315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206448322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/94da9c146d1b4fe49357e3b0517ea7af is 50, key is test_row_0/A:col10/1733206388169/Put/seqid=0 2024-12-03T06:13:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741837_1013 (size=16681) 2024-12-03T06:13:08,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/94da9c146d1b4fe49357e3b0517ea7af 2024-12-03T06:13:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T06:13:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206448481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206448482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206448483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/064cfab7404042f1a6a94fb028d7d533 is 50, key is test_row_0/B:col10/1733206388169/Put/seqid=0 2024-12-03T06:13:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T06:13:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206448484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206448484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741838_1014 (size=12001) 2024-12-03T06:13:08,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/064cfab7404042f1a6a94fb028d7d533 2024-12-03T06:13:08,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6bba559202f345db9680f5991ff83ec4 is 50, key is test_row_0/C:col10/1733206388169/Put/seqid=0 2024-12-03T06:13:08,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741839_1015 (size=12001) 2024-12-03T06:13:08,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6bba559202f345db9680f5991ff83ec4 2024-12-03T06:13:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/94da9c146d1b4fe49357e3b0517ea7af as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af 2024-12-03T06:13:08,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T06:13:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af, entries=250, sequenceid=15, filesize=16.3 K 2024-12-03T06:13:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/064cfab7404042f1a6a94fb028d7d533 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533 2024-12-03T06:13:08,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206448693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T06:13:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206448694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206448694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T06:13:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206448701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6bba559202f345db9680f5991ff83ec4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4 2024-12-03T06:13:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206448706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:08,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T06:13:08,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8ec9e85614363b6e4e059ef6575a1df6 in 573ms, sequenceid=15, compaction requested=false 2024-12-03T06:13:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T06:13:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:08,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T06:13:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:08,830 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:08,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/830ced3f013143f5b35d9b95f8b809bd is 50, key is test_row_0/A:col10/1733206388299/Put/seqid=0 2024-12-03T06:13:08,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741840_1016 (size=12001) 2024-12-03T06:13:08,863 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/830ced3f013143f5b35d9b95f8b809bd 2024-12-03T06:13:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/85fa72c33c6e48a0808548549cee9126 is 50, key is test_row_0/B:col10/1733206388299/Put/seqid=0 2024-12-03T06:13:08,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741841_1017 (size=12001) 2024-12-03T06:13:08,950 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/85fa72c33c6e48a0808548549cee9126 2024-12-03T06:13:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/25989e22e30c4aef83b8d9c204276128 is 50, key is test_row_0/C:col10/1733206388299/Put/seqid=0 2024-12-03T06:13:09,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:09,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741842_1018 (size=12001) 2024-12-03T06:13:09,043 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/25989e22e30c4aef83b8d9c204276128 2024-12-03T06:13:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/830ced3f013143f5b35d9b95f8b809bd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd 2024-12-03T06:13:09,072 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd, entries=150, sequenceid=33, filesize=11.7 K 2024-12-03T06:13:09,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/85fa72c33c6e48a0808548549cee9126 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126 2024-12-03T06:13:09,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206449069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,090 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126, entries=150, sequenceid=33, filesize=11.7 K 2024-12-03T06:13:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/25989e22e30c4aef83b8d9c204276128 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128 2024-12-03T06:13:09,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206449071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206449077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206449078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206449081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128, entries=150, sequenceid=33, filesize=11.7 K 2024-12-03T06:13:09,114 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8ec9e85614363b6e4e059ef6575a1df6 in 285ms, sequenceid=33, compaction requested=false 2024-12-03T06:13:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T06:13:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T06:13:09,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T06:13:09,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-12-03T06:13:09,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0850 sec 2024-12-03T06:13:09,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:13:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:09,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:09,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0bd8bfc1f81845debd3e08bdec2a1f6d is 50, key is test_row_0/A:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T06:13:09,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:09,223 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T06:13:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T06:13:09,227 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:09,232 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:09,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:09,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741843_1019 (size=14341) 2024-12-03T06:13:09,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0bd8bfc1f81845debd3e08bdec2a1f6d 2024-12-03T06:13:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206449263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206449273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206449274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206449276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206449285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fb09d05b29b9467e82caacde7c7953f4 is 50, key is test_row_0/B:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741844_1020 (size=12001) 2024-12-03T06:13:09,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fb09d05b29b9467e82caacde7c7953f4 2024-12-03T06:13:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T06:13:09,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/405fdabc3f624a9abc23666992c7ab5c is 50, key is test_row_0/C:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T06:13:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:09,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206449390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206449401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206449401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206449402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206449402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741845_1021 (size=12001) 2024-12-03T06:13:09,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/405fdabc3f624a9abc23666992c7ab5c 2024-12-03T06:13:09,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0bd8bfc1f81845debd3e08bdec2a1f6d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d 2024-12-03T06:13:09,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d, entries=200, sequenceid=49, filesize=14.0 K 2024-12-03T06:13:09,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fb09d05b29b9467e82caacde7c7953f4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4 2024-12-03T06:13:09,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T06:13:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/405fdabc3f624a9abc23666992c7ab5c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c 2024-12-03T06:13:09,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T06:13:09,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8ec9e85614363b6e4e059ef6575a1df6 in 311ms, sequenceid=49, compaction requested=true 2024-12-03T06:13:09,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:09,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:09,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:09,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:09,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:09,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:09,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:09,529 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:09,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:09,536 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:09,538 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:09,539 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,539 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.2 K 2024-12-03T06:13:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T06:13:09,542 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 064cfab7404042f1a6a94fb028d7d533, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206388131 2024-12-03T06:13:09,542 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:09,542 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:09,542 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,543 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=42.0 K 2024-12-03T06:13:09,544 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 85fa72c33c6e48a0808548549cee9126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733206388293 2024-12-03T06:13:09,544 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94da9c146d1b4fe49357e3b0517ea7af, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206388116 2024-12-03T06:13:09,545 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting fb09d05b29b9467e82caacde7c7953f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389066 2024-12-03T06:13:09,545 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830ced3f013143f5b35d9b95f8b809bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733206388293 2024-12-03T06:13:09,546 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd8bfc1f81845debd3e08bdec2a1f6d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389051 2024-12-03T06:13:09,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T06:13:09,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,549 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:09,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/067373fd734b44e4aeea0984449728f6 is 50, key is test_row_0/A:col10/1733206389263/Put/seqid=0 2024-12-03T06:13:09,603 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:09,606 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/f4d4fce017d64d5d865e1c9a556d1736 is 50, key is test_row_0/B:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741846_1022 (size=12001) 2024-12-03T06:13:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:09,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:09,633 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#11 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:09,634 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6593eff2d2e041e097670d1e77ee9652 is 50, key is test_row_0/A:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741848_1024 (size=12104) 2024-12-03T06:13:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741847_1023 (size=12104) 2024-12-03T06:13:09,671 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6593eff2d2e041e097670d1e77ee9652 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6593eff2d2e041e097670d1e77ee9652 2024-12-03T06:13:09,678 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/f4d4fce017d64d5d865e1c9a556d1736 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f4d4fce017d64d5d865e1c9a556d1736 2024-12-03T06:13:09,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206449688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206449695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206449691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206449701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,711 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into f4d4fce017d64d5d865e1c9a556d1736(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:09,711 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 6593eff2d2e041e097670d1e77ee9652(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:09,711 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:09,711 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206449703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,714 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206389510; duration=0sec 2024-12-03T06:13:09,714 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206389527; duration=0sec 2024-12-03T06:13:09,715 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:09,715 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:09,715 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:09,716 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:09,715 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:09,719 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:09,719 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:09,719 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:09,719 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.2 K 2024-12-03T06:13:09,721 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bba559202f345db9680f5991ff83ec4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206388131 2024-12-03T06:13:09,723 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 25989e22e30c4aef83b8d9c204276128, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733206388293 2024-12-03T06:13:09,724 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 405fdabc3f624a9abc23666992c7ab5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389066 2024-12-03T06:13:09,786 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:09,795 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/853e760848ae48468477faf47d695fca is 50, key is test_row_0/C:col10/1733206389074/Put/seqid=0 2024-12-03T06:13:09,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206449811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206449821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741849_1025 (size=12104) 2024-12-03T06:13:09,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206449820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206449822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206449821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T06:13:09,854 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/853e760848ae48468477faf47d695fca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/853e760848ae48468477faf47d695fca 2024-12-03T06:13:09,878 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 853e760848ae48468477faf47d695fca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:09,879 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:09,879 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206389527; duration=0sec 2024-12-03T06:13:09,879 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:09,879 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:10,015 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/067373fd734b44e4aeea0984449728f6 2024-12-03T06:13:10,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206450020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/6778a8e261e24fbb8059f5e2b46340ec is 50, key is test_row_0/B:col10/1733206389263/Put/seqid=0 2024-12-03T06:13:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206450042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741850_1026 (size=12001) 2024-12-03T06:13:10,084 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/6778a8e261e24fbb8059f5e2b46340ec 2024-12-03T06:13:10,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/375b0735afaf47b4942e5b7f0a8d0eba is 50, key is test_row_0/C:col10/1733206389263/Put/seqid=0 2024-12-03T06:13:10,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741851_1027 (size=12001) 2024-12-03T06:13:10,160 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/375b0735afaf47b4942e5b7f0a8d0eba 2024-12-03T06:13:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/067373fd734b44e4aeea0984449728f6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6 2024-12-03T06:13:10,199 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T06:13:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/6778a8e261e24fbb8059f5e2b46340ec as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec 2024-12-03T06:13:10,221 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T06:13:10,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/375b0735afaf47b4942e5b7f0a8d0eba as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba 2024-12-03T06:13:10,242 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T06:13:10,245 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8ec9e85614363b6e4e059ef6575a1df6 in 695ms, sequenceid=65, compaction requested=false 2024-12-03T06:13:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T06:13:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T06:13:10,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T06:13:10,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-12-03T06:13:10,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0350 sec 2024-12-03T06:13:10,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T06:13:10,361 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:10,363 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:10,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/64b0e7fbfa5b421fb15f109ca002dc51 is 50, key is test_row_0/A:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T06:13:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:10,418 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:10,420 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:10,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741852_1028 (size=12001) 2024-12-03T06:13:10,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/64b0e7fbfa5b421fb15f109ca002dc51 2024-12-03T06:13:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206450424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206450437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206450441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206450441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206450441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d4bc7602014b4fa5bb84df1384686dcc is 50, key is test_row_0/B:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:10,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741853_1029 (size=12001) 2024-12-03T06:13:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206450551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206450551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206450560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206450560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206450760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206450760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206450770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206450780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206450780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:10,815 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:13:10,883 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T06:13:10,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:10,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d4bc7602014b4fa5bb84df1384686dcc 2024-12-03T06:13:10,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:10,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:10,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:10,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/a67077dd959c4af29cb0b35da8e9e34c is 50, key is test_row_0/C:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:10,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741854_1030 (size=12001) 2024-12-03T06:13:11,048 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T06:13:11,048 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T06:13:11,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T06:13:11,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T06:13:11,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T06:13:11,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T06:13:11,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T06:13:11,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T06:13:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:11,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:11,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206451070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206451070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206451080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206451090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206451090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:11,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/a67077dd959c4af29cb0b35da8e9e34c 2024-12-03T06:13:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/64b0e7fbfa5b421fb15f109ca002dc51 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51 2024-12-03T06:13:11,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T06:13:11,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d4bc7602014b4fa5bb84df1384686dcc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc 2024-12-03T06:13:11,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T06:13:11,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/a67077dd959c4af29cb0b35da8e9e34c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c 2024-12-03T06:13:11,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T06:13:11,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8ec9e85614363b6e4e059ef6575a1df6 in 1129ms, sequenceid=86, compaction requested=true 2024-12-03T06:13:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:11,469 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:11,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:13:11,469 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:11,472 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:11,472 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:11,472 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:11,472 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:11,472 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,472 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,473 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f4d4fce017d64d5d865e1c9a556d1736, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.3 K 2024-12-03T06:13:11,473 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6593eff2d2e041e097670d1e77ee9652, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.3 K 2024-12-03T06:13:11,474 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6593eff2d2e041e097670d1e77ee9652, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389066 2024-12-03T06:13:11,474 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d4fce017d64d5d865e1c9a556d1736, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389066 2024-12-03T06:13:11,475 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 067373fd734b44e4aeea0984449728f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206389247 2024-12-03T06:13:11,475 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6778a8e261e24fbb8059f5e2b46340ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206389247 2024-12-03T06:13:11,476 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bc7602014b4fa5bb84df1384686dcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:11,476 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b0e7fbfa5b421fb15f109ca002dc51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:11,503 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:11,503 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:11,504 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/142389f539af4290a35e3928aa4ff565 is 50, key is test_row_0/B:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:11,504 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/bcfaba1bc3d848899bc9f72872d6df53 is 50, key is test_row_0/A:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:11,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T06:13:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,535 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:11,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741855_1031 (size=12207) 2024-12-03T06:13:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/806a40e908094d64816be627f951effa is 50, key is test_row_0/A:col10/1733206390435/Put/seqid=0 2024-12-03T06:13:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:11,585 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/bcfaba1bc3d848899bc9f72872d6df53 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bcfaba1bc3d848899bc9f72872d6df53 2024-12-03T06:13:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741856_1032 (size=12207) 2024-12-03T06:13:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:11,612 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/142389f539af4290a35e3928aa4ff565 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/142389f539af4290a35e3928aa4ff565 2024-12-03T06:13:11,614 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into bcfaba1bc3d848899bc9f72872d6df53(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:11,615 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:11,615 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206391468; duration=0sec 2024-12-03T06:13:11,616 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:11,616 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:11,616 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:11,628 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 142389f539af4290a35e3928aa4ff565(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:11,628 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:11,629 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206391469; duration=0sec 2024-12-03T06:13:11,629 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:11,629 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:11,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741857_1033 (size=12001) 2024-12-03T06:13:11,631 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/806a40e908094d64816be627f951effa 2024-12-03T06:13:11,634 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:11,635 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:11,635 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:11,637 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/853e760848ae48468477faf47d695fca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.3 K 2024-12-03T06:13:11,640 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853e760848ae48468477faf47d695fca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733206389066 2024-12-03T06:13:11,643 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 375b0735afaf47b4942e5b7f0a8d0eba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206389247 2024-12-03T06:13:11,648 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67077dd959c4af29cb0b35da8e9e34c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206451635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206451642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206451646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206451653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206451657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a9c90eff71b44c348b63981cafb5beb4 is 50, key is test_row_0/B:col10/1733206390435/Put/seqid=0 2024-12-03T06:13:11,672 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:11,673 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8212307d311b4e689af1b478c79da6d9 is 50, key is test_row_0/C:col10/1733206389694/Put/seqid=0 2024-12-03T06:13:11,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741859_1035 (size=12207) 2024-12-03T06:13:11,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741858_1034 (size=12001) 2024-12-03T06:13:11,726 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a9c90eff71b44c348b63981cafb5beb4 2024-12-03T06:13:11,729 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8212307d311b4e689af1b478c79da6d9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8212307d311b4e689af1b478c79da6d9 2024-12-03T06:13:11,761 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 8212307d311b4e689af1b478c79da6d9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:11,761 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:11,761 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206391469; duration=0sec 2024-12-03T06:13:11,762 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:11,762 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:11,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206451761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206451761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206451762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/94639ab2f8e540389c72a712a25fd010 is 50, key is test_row_0/C:col10/1733206390435/Put/seqid=0 2024-12-03T06:13:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206451770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206451774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741860_1036 (size=12001) 2024-12-03T06:13:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206451980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206451980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206451982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206451989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206451990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,215 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/94639ab2f8e540389c72a712a25fd010 2024-12-03T06:13:12,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/806a40e908094d64816be627f951effa as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa 2024-12-03T06:13:12,247 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T06:13:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a9c90eff71b44c348b63981cafb5beb4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4 2024-12-03T06:13:12,263 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T06:13:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/94639ab2f8e540389c72a712a25fd010 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010 2024-12-03T06:13:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T06:13:12,287 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8ec9e85614363b6e4e059ef6575a1df6 in 753ms, sequenceid=101, compaction requested=false 2024-12-03T06:13:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T06:13:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T06:13:12,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T06:13:12,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-03T06:13:12,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9330 sec 2024-12-03T06:13:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:12,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:12,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b65fe407f81f4cb0a14d7b2cacc6d474 is 50, key is test_row_0/A:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206452348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206452348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206452353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206452360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206452363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741861_1037 (size=12001) 2024-12-03T06:13:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206452470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206452470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206452481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206452482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206452482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T06:13:12,580 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:12,581 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T06:13:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:12,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:12,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:12,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:12,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206452680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:12,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206452692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206452692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206452693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206452693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:12,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:12,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b65fe407f81f4cb0a14d7b2cacc6d474 2024-12-03T06:13:12,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/eac98821147d402ea1b1a39cbc8c2669 is 50, key is test_row_0/B:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:12,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741862_1038 (size=12001) 2024-12-03T06:13:12,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/eac98821147d402ea1b1a39cbc8c2669 2024-12-03T06:13:12,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1b99d31090d44e59b622e483725ba592 is 50, key is test_row_0/C:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:12,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:12,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:12,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:12,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:12,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741863_1039 (size=12001) 2024-12-03T06:13:13,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206453000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206453001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206453011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206453010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206453016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:13,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1b99d31090d44e59b622e483725ba592 2024-12-03T06:13:13,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b65fe407f81f4cb0a14d7b2cacc6d474 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474 2024-12-03T06:13:13,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T06:13:13,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/eac98821147d402ea1b1a39cbc8c2669 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669 2024-12-03T06:13:13,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T06:13:13,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1b99d31090d44e59b622e483725ba592 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592 2024-12-03T06:13:13,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T06:13:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ec9e85614363b6e4e059ef6575a1df6 in 1071ms, sequenceid=123, compaction requested=true 2024-12-03T06:13:13,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:13,375 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:13,375 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:13,377 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:13,377 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:13,377 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,377 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:13,377 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bcfaba1bc3d848899bc9f72872d6df53, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.4 K 2024-12-03T06:13:13,377 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:13,377 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,378 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/142389f539af4290a35e3928aa4ff565, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.4 K 2024-12-03T06:13:13,378 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfaba1bc3d848899bc9f72872d6df53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:13,378 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 142389f539af4290a35e3928aa4ff565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:13,378 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806a40e908094d64816be627f951effa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733206390434 2024-12-03T06:13:13,379 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c90eff71b44c348b63981cafb5beb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733206390434 2024-12-03T06:13:13,379 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b65fe407f81f4cb0a14d7b2cacc6d474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:13,379 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting eac98821147d402ea1b1a39cbc8c2669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:13,397 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:13,398 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/10de59982c9144229aee2e2eccdafc52 is 50, key is test_row_0/A:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:13,412 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:13,413 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/474c69911e9f4240a39773284526c64d is 50, key is test_row_0/B:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:13,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741864_1040 (size=12309) 2024-12-03T06:13:13,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741865_1041 (size=12309) 2024-12-03T06:13:13,442 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/10de59982c9144229aee2e2eccdafc52 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/10de59982c9144229aee2e2eccdafc52 2024-12-03T06:13:13,465 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/474c69911e9f4240a39773284526c64d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/474c69911e9f4240a39773284526c64d 2024-12-03T06:13:13,467 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 10de59982c9144229aee2e2eccdafc52(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:13,467 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:13,467 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206393374; duration=0sec 2024-12-03T06:13:13,467 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:13,467 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:13,470 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:13,473 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:13,474 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:13,474 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,474 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8212307d311b4e689af1b478c79da6d9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=35.4 K 2024-12-03T06:13:13,475 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8212307d311b4e689af1b478c79da6d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733206389694 2024-12-03T06:13:13,476 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94639ab2f8e540389c72a712a25fd010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733206390434 2024-12-03T06:13:13,477 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b99d31090d44e59b622e483725ba592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:13,485 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 474c69911e9f4240a39773284526c64d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:13,485 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:13,485 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206393375; duration=0sec 2024-12-03T06:13:13,485 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:13,485 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:13,495 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:13,496 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0591f79c972f47988ab7f365a91b9c89 is 50, key is test_row_0/C:col10/1733206392302/Put/seqid=0 2024-12-03T06:13:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:13,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:13,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b4c0384da8fc49f9bd1758b2812e0686 is 50, key is test_row_0/A:col10/1733206392352/Put/seqid=0 2024-12-03T06:13:13,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741866_1042 (size=12309) 2024-12-03T06:13:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206453545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206453547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206453549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206453556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206453556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741867_1043 (size=12151) 2024-12-03T06:13:13,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b4c0384da8fc49f9bd1758b2812e0686 2024-12-03T06:13:13,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cdf2bacc5a2c4305b16285c8338cf571 is 50, key is test_row_0/B:col10/1733206392352/Put/seqid=0 2024-12-03T06:13:13,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741868_1044 (size=12151) 2024-12-03T06:13:13,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206453660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206453660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206453662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206453670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206453671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:13,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206453870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206453870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206453870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206453880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206453880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:13,971 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0591f79c972f47988ab7f365a91b9c89 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0591f79c972f47988ab7f365a91b9c89 2024-12-03T06:13:13,992 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 0591f79c972f47988ab7f365a91b9c89(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:13,992 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:13,993 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206393375; duration=0sec 2024-12-03T06:13:13,993 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:13,993 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:13,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:13,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:13,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:13,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:14,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cdf2bacc5a2c4305b16285c8338cf571 2024-12-03T06:13:14,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2c4aef6a6a3842c0b8e0a0bb65324768 is 50, key is test_row_0/C:col10/1733206392352/Put/seqid=0 2024-12-03T06:13:14,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741869_1045 (size=12151) 2024-12-03T06:13:14,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2c4aef6a6a3842c0b8e0a0bb65324768 2024-12-03T06:13:14,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b4c0384da8fc49f9bd1758b2812e0686 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686 2024-12-03T06:13:14,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686, entries=150, sequenceid=139, filesize=11.9 K 2024-12-03T06:13:14,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cdf2bacc5a2c4305b16285c8338cf571 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571 2024-12-03T06:13:14,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571, entries=150, sequenceid=139, filesize=11.9 K 2024-12-03T06:13:14,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2c4aef6a6a3842c0b8e0a0bb65324768 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768 2024-12-03T06:13:14,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768, entries=150, sequenceid=139, filesize=11.9 K 2024-12-03T06:13:14,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 8ec9e85614363b6e4e059ef6575a1df6 in 600ms, sequenceid=139, compaction requested=false 2024-12-03T06:13:14,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:14,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T06:13:14,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:14,152 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3dbf8498206544f6a0f100037835c32a is 50, key is test_row_0/A:col10/1733206393545/Put/seqid=0 2024-12-03T06:13:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:14,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:14,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741870_1046 (size=12151) 2024-12-03T06:13:14,205 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3dbf8498206544f6a0f100037835c32a 2024-12-03T06:13:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206454199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206454195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206454202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206454199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206454207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fe55d5bcf2754e01bd04b648c523813a is 50, key is test_row_0/B:col10/1733206393545/Put/seqid=0 2024-12-03T06:13:14,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741871_1047 (size=12151) 2024-12-03T06:13:14,244 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fe55d5bcf2754e01bd04b648c523813a 2024-12-03T06:13:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1579ac4f959241478c2708043607a60b is 50, key is test_row_0/C:col10/1733206393545/Put/seqid=0 2024-12-03T06:13:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741872_1048 (size=12151) 2024-12-03T06:13:14,283 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1579ac4f959241478c2708043607a60b 2024-12-03T06:13:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3dbf8498206544f6a0f100037835c32a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a 2024-12-03T06:13:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206454310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206454310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206454311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206454320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206454320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,328 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a, entries=150, sequenceid=159, filesize=11.9 K 2024-12-03T06:13:14,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/fe55d5bcf2754e01bd04b648c523813a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a 2024-12-03T06:13:14,347 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a, entries=150, sequenceid=159, filesize=11.9 K 2024-12-03T06:13:14,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1579ac4f959241478c2708043607a60b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b 2024-12-03T06:13:14,361 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b, entries=150, sequenceid=159, filesize=11.9 K 2024-12-03T06:13:14,363 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ec9e85614363b6e4e059ef6575a1df6 in 211ms, sequenceid=159, compaction requested=true 2024-12-03T06:13:14,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:14,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:14,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T06:13:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T06:13:14,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T06:13:14,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-12-03T06:13:14,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8070 sec 2024-12-03T06:13:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:14,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:14,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/36b1fe0753bb4272b1d29e63f6848df2 is 50, key is test_row_0/A:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:14,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741873_1049 (size=12151) 2024-12-03T06:13:14,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206454548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206454550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206454552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206454554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206454554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206454660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206454660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206454660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206454669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206454670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T06:13:14,730 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:14,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T06:13:14,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T06:13:14,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:14,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T06:13:14,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206454870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206454870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206454870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206454880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206454880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:14,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T06:13:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:14,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:14,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/36b1fe0753bb4272b1d29e63f6848df2 2024-12-03T06:13:14,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cde5fb072a7245e891a99c8ab7f7c1a2 is 50, key is test_row_0/B:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:14,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741874_1050 (size=12151) 2024-12-03T06:13:14,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cde5fb072a7245e891a99c8ab7f7c1a2 2024-12-03T06:13:14,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ed8f07eebf7349e59b1ad7340b40d841 is 50, key is test_row_0/C:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:15,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741875_1051 (size=12151) 2024-12-03T06:13:15,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T06:13:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:15,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T06:13:15,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206455179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206455180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206455180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206455190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206455190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T06:13:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T06:13:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T06:13:15,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ed8f07eebf7349e59b1ad7340b40d841 2024-12-03T06:13:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/36b1fe0753bb4272b1d29e63f6848df2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2 2024-12-03T06:13:15,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T06:13:15,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cde5fb072a7245e891a99c8ab7f7c1a2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2 2024-12-03T06:13:15,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T06:13:15,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ed8f07eebf7349e59b1ad7340b40d841 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841 2024-12-03T06:13:15,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T06:13:15,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8ec9e85614363b6e4e059ef6575a1df6 in 958ms, sequenceid=173, compaction requested=true 2024-12-03T06:13:15,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:15,490 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:15,490 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:15,492 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:15,492 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:15,492 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:15,492 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:15,492 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,492 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,492 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/10de59982c9144229aee2e2eccdafc52, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=47.6 K 2024-12-03T06:13:15,492 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/474c69911e9f4240a39773284526c64d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=47.6 K 2024-12-03T06:13:15,493 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 474c69911e9f4240a39773284526c64d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:15,493 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10de59982c9144229aee2e2eccdafc52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:15,494 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf2bacc5a2c4305b16285c8338cf571, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733206392352 2024-12-03T06:13:15,494 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c0384da8fc49f9bd1758b2812e0686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733206392352 2024-12-03T06:13:15,494 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbf8498206544f6a0f100037835c32a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733206393545 2024-12-03T06:13:15,494 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting fe55d5bcf2754e01bd04b648c523813a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733206393545 2024-12-03T06:13:15,495 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b1fe0753bb4272b1d29e63f6848df2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:15,495 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cde5fb072a7245e891a99c8ab7f7c1a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:15,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T06:13:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,509 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6bd7e6e2d0684d8cb56ec57a6be5055f is 50, key is test_row_0/A:col10/1733206394543/Put/seqid=0 2024-12-03T06:13:15,530 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:15,530 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:15,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b582add2ffcf4adca3cfc95521e090be is 50, key is test_row_0/A:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:15,530 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/dea8aefe824f40c9b3b83095eb4bf9fe is 50, key is test_row_0/B:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:15,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741877_1053 (size=12595) 2024-12-03T06:13:15,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741876_1052 (size=12151) 2024-12-03T06:13:15,555 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6bd7e6e2d0684d8cb56ec57a6be5055f 2024-12-03T06:13:15,569 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b582add2ffcf4adca3cfc95521e090be as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b582add2ffcf4adca3cfc95521e090be 2024-12-03T06:13:15,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741878_1054 (size=12595) 2024-12-03T06:13:15,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0f20cf2c233413ebc93d1904c4a643d is 50, key is test_row_0/B:col10/1733206394543/Put/seqid=0 2024-12-03T06:13:15,588 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/dea8aefe824f40c9b3b83095eb4bf9fe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/dea8aefe824f40c9b3b83095eb4bf9fe 2024-12-03T06:13:15,591 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into b582add2ffcf4adca3cfc95521e090be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:15,591 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:15,591 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=12, startTime=1733206395490; duration=0sec 2024-12-03T06:13:15,593 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:15,593 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:15,593 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:15,596 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:15,596 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:15,597 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,597 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0591f79c972f47988ab7f365a91b9c89, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=47.6 K 2024-12-03T06:13:15,598 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0591f79c972f47988ab7f365a91b9c89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733206391636 2024-12-03T06:13:15,598 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4aef6a6a3842c0b8e0a0bb65324768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733206392352 2024-12-03T06:13:15,599 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1579ac4f959241478c2708043607a60b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733206393545 2024-12-03T06:13:15,599 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8f07eebf7349e59b1ad7340b40d841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:15,615 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into dea8aefe824f40c9b3b83095eb4bf9fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:15,615 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:15,615 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=12, startTime=1733206395490; duration=0sec 2024-12-03T06:13:15,615 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:15,615 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:15,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741879_1055 (size=12151) 2024-12-03T06:13:15,637 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0f20cf2c233413ebc93d1904c4a643d 2024-12-03T06:13:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/edc2bfd7e4bd481b82b7381198630d71 is 50, key is test_row_0/C:col10/1733206394543/Put/seqid=0 2024-12-03T06:13:15,675 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:15,676 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6c23a1a1f18e4fddb05f18775b70d4b5 is 50, key is test_row_0/C:col10/1733206394530/Put/seqid=0 2024-12-03T06:13:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:15,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741880_1056 (size=12151) 2024-12-03T06:13:15,723 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/edc2bfd7e4bd481b82b7381198630d71 2024-12-03T06:13:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206455716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206455716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206455720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206455718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206455725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6bd7e6e2d0684d8cb56ec57a6be5055f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f 2024-12-03T06:13:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741881_1057 (size=12595) 2024-12-03T06:13:15,748 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T06:13:15,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0f20cf2c233413ebc93d1904c4a643d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d 2024-12-03T06:13:15,781 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T06:13:15,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/edc2bfd7e4bd481b82b7381198630d71 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71 2024-12-03T06:13:15,795 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T06:13:15,796 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8ec9e85614363b6e4e059ef6575a1df6 in 287ms, sequenceid=191, compaction requested=false 2024-12-03T06:13:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T06:13:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T06:13:15,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T06:13:15,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-12-03T06:13:15,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0780 sec 2024-12-03T06:13:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:15,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:13:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:15,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:15,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6cb7d93656234815be8cb3b8c535af47 is 50, key is test_row_0/A:col10/1733206395843/Put/seqid=0 2024-12-03T06:13:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T06:13:15,870 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:15,871 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T06:13:15,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206455868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206455868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206455870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T06:13:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206455874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206455875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741882_1058 (size=12151) 2024-12-03T06:13:15,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6cb7d93656234815be8cb3b8c535af47 2024-12-03T06:13:15,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/669511ff1f3a4808a333843853d6274e is 50, key is test_row_0/B:col10/1733206395843/Put/seqid=0 2024-12-03T06:13:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741883_1059 (size=12151) 2024-12-03T06:13:15,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/669511ff1f3a4808a333843853d6274e 2024-12-03T06:13:15,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0904fce9405f45afb4d7c42cbff3337d is 50, key is test_row_0/C:col10/1733206395843/Put/seqid=0 2024-12-03T06:13:15,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741884_1060 (size=12151) 2024-12-03T06:13:15,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0904fce9405f45afb4d7c42cbff3337d 2024-12-03T06:13:15,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6cb7d93656234815be8cb3b8c535af47 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47 2024-12-03T06:13:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T06:13:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206455980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T06:13:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206455980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/669511ff1f3a4808a333843853d6274e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e 2024-12-03T06:13:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206455980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T06:13:15,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0904fce9405f45afb4d7c42cbff3337d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d 2024-12-03T06:13:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206455990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206455991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T06:13:16,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8ec9e85614363b6e4e059ef6575a1df6 in 158ms, sequenceid=211, compaction requested=true 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:16,004 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:13:16,006 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:16,006 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:16,006 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:16,006 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b582add2ffcf4adca3cfc95521e090be, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.0 K 2024-12-03T06:13:16,007 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b582add2ffcf4adca3cfc95521e090be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:16,008 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd7e6e2d0684d8cb56ec57a6be5055f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733206394543 2024-12-03T06:13:16,008 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb7d93656234815be8cb3b8c535af47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733206395843 2024-12-03T06:13:16,022 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:16,023 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a3bef26f0c1240758ae4e262073b2bb2 is 50, key is test_row_0/A:col10/1733206395843/Put/seqid=0 2024-12-03T06:13:16,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T06:13:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:16,029 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4795e72b5b7747e1b1ea72b8cfae8db0 is 50, key is test_row_0/A:col10/1733206395868/Put/seqid=0 2024-12-03T06:13:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741885_1061 (size=12697) 2024-12-03T06:13:16,066 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a3bef26f0c1240758ae4e262073b2bb2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a3bef26f0c1240758ae4e262073b2bb2 2024-12-03T06:13:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741886_1062 (size=12151) 2024-12-03T06:13:16,082 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into a3bef26f0c1240758ae4e262073b2bb2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:16,083 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:16,083 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206396004; duration=0sec 2024-12-03T06:13:16,083 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:13:16,083 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:16,083 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-12-03T06:13:16,084 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T06:13:16,085 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T06:13:16,085 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. because compaction request was cancelled 2024-12-03T06:13:16,086 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:16,086 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:16,087 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:16,087 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:16,087 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:16,088 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/dea8aefe824f40c9b3b83095eb4bf9fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.0 K 2024-12-03T06:13:16,088 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting dea8aefe824f40c9b3b83095eb4bf9fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:16,089 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f20cf2c233413ebc93d1904c4a643d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733206394543 2024-12-03T06:13:16,089 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 669511ff1f3a4808a333843853d6274e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733206395843 2024-12-03T06:13:16,101 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:16,101 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0ffde0dc2d864560a5ad0ec7aa5e9877 is 50, key is test_row_0/B:col10/1733206395843/Put/seqid=0 2024-12-03T06:13:16,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741887_1063 (size=12697) 2024-12-03T06:13:16,166 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6c23a1a1f18e4fddb05f18775b70d4b5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6c23a1a1f18e4fddb05f18775b70d4b5 2024-12-03T06:13:16,179 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 6c23a1a1f18e4fddb05f18775b70d4b5(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:16,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:16,179 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=12, startTime=1733206395490; duration=0sec 2024-12-03T06:13:16,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:16,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T06:13:16,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:16,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206456215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206456222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206456256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206456257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206456257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206456369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206456370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206456370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206456370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206456370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,481 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4795e72b5b7747e1b1ea72b8cfae8db0 2024-12-03T06:13:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0854e18e65b143339e39c28c5761c177 is 50, key is test_row_0/B:col10/1733206395868/Put/seqid=0 2024-12-03T06:13:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T06:13:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741888_1064 (size=12151) 2024-12-03T06:13:16,532 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0ffde0dc2d864560a5ad0ec7aa5e9877 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0ffde0dc2d864560a5ad0ec7aa5e9877 2024-12-03T06:13:16,539 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 0ffde0dc2d864560a5ad0ec7aa5e9877(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:16,539 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:16,539 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206396004; duration=0sec 2024-12-03T06:13:16,539 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:16,539 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206456579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206456579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206456580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206456580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206456580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206456890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206456890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206456890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206456890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206456890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:16,903 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0854e18e65b143339e39c28c5761c177 2024-12-03T06:13:16,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1535663b3abf4c01a7c968b03d03cdef is 50, key is test_row_0/C:col10/1733206395868/Put/seqid=0 2024-12-03T06:13:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741889_1065 (size=12151) 2024-12-03T06:13:16,919 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1535663b3abf4c01a7c968b03d03cdef 2024-12-03T06:13:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4795e72b5b7747e1b1ea72b8cfae8db0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0 2024-12-03T06:13:16,933 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T06:13:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0854e18e65b143339e39c28c5761c177 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177 2024-12-03T06:13:16,941 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T06:13:16,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/1535663b3abf4c01a7c968b03d03cdef as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef 2024-12-03T06:13:16,956 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T06:13:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 8ec9e85614363b6e4e059ef6575a1df6 in 928ms, sequenceid=226, compaction requested=true 2024-12-03T06:13:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T06:13:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T06:13:16,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T06:13:16,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-12-03T06:13:16,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0950 sec 2024-12-03T06:13:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T06:13:17,010 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:17,010 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T06:13:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T06:13:17,013 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:17,013 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:17,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T06:13:17,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T06:13:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:17,166 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/1a514a1f225a464286b184099fa65c0e is 50, key is test_row_0/A:col10/1733206396219/Put/seqid=0 2024-12-03T06:13:17,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741890_1066 (size=12151) 2024-12-03T06:13:17,212 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/1a514a1f225a464286b184099fa65c0e 2024-12-03T06:13:17,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cec7f3867f4a4846ab35e0a0cfe7933c is 50, key is test_row_0/B:col10/1733206396219/Put/seqid=0 2024-12-03T06:13:17,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741891_1067 (size=12151) 2024-12-03T06:13:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T06:13:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206457428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206457428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206457431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206457433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206457431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206457549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206457549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206457549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206457550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206457550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T06:13:17,647 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cec7f3867f4a4846ab35e0a0cfe7933c 2024-12-03T06:13:17,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ad80fcd55ba4426a90f7b6316769c79e is 50, key is test_row_0/C:col10/1733206396219/Put/seqid=0 2024-12-03T06:13:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741892_1068 (size=12151) 2024-12-03T06:13:17,662 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ad80fcd55ba4426a90f7b6316769c79e 2024-12-03T06:13:17,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/1a514a1f225a464286b184099fa65c0e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e 2024-12-03T06:13:17,677 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T06:13:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/cec7f3867f4a4846ab35e0a0cfe7933c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c 2024-12-03T06:13:17,685 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T06:13:17,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/ad80fcd55ba4426a90f7b6316769c79e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e 2024-12-03T06:13:17,691 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T06:13:17,692 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8ec9e85614363b6e4e059ef6575a1df6 in 526ms, sequenceid=248, compaction requested=true 2024-12-03T06:13:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T06:13:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T06:13:17,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T06:13:17,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-12-03T06:13:17,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 688 msec 2024-12-03T06:13:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:17,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:17,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/8414b7acca30402d95ecc0b7edd21c5d is 50, key is test_row_0/A:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:17,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741893_1069 (size=17131) 2024-12-03T06:13:17,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206457787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206457788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206457789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206457793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206457794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206457899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206457899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206457899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206457900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:17,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206457910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206458110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206458110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206458110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206458110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206458119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T06:13:18,150 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:18,151 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-03T06:13:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:18,153 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:18,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:18,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:18,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/8414b7acca30402d95ecc0b7edd21c5d 2024-12-03T06:13:18,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/582123bae957447388d22835e4cdf2a4 is 50, key is test_row_0/B:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:18,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741894_1070 (size=12301) 2024-12-03T06:13:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:18,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206458419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206458419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206458419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206458420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206458430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:18,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/582123bae957447388d22835e4cdf2a4 2024-12-03T06:13:18,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8e95d020d4b146e78a6865be805c8e5c is 50, key is test_row_0/C:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:18,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741895_1071 (size=12301) 2024-12-03T06:13:18,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:18,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:18,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:18,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206458930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206458930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206458930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206458930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:18,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206458940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:19,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8e95d020d4b146e78a6865be805c8e5c 2024-12-03T06:13:19,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/8414b7acca30402d95ecc0b7edd21c5d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d 2024-12-03T06:13:19,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d, entries=250, sequenceid=263, filesize=16.7 K 2024-12-03T06:13:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/582123bae957447388d22835e4cdf2a4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4 2024-12-03T06:13:19,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T06:13:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8e95d020d4b146e78a6865be805c8e5c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c 2024-12-03T06:13:19,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T06:13:19,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8ec9e85614363b6e4e059ef6575a1df6 in 1303ms, sequenceid=263, compaction requested=true 2024-12-03T06:13:19,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:19,066 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:19,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:19,067 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:19,067 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:19,067 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:19,068 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:19,068 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a3bef26f0c1240758ae4e262073b2bb2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=52.9 K 2024-12-03T06:13:19,068 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:19,068 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3bef26f0c1240758ae4e262073b2bb2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733206395843 2024-12-03T06:13:19,068 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:19,068 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:19,068 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4795e72b5b7747e1b1ea72b8cfae8db0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733206395866 2024-12-03T06:13:19,068 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0ffde0dc2d864560a5ad0ec7aa5e9877, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=48.1 K 2024-12-03T06:13:19,069 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a514a1f225a464286b184099fa65c0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733206396211 2024-12-03T06:13:19,069 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ffde0dc2d864560a5ad0ec7aa5e9877, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733206395843 2024-12-03T06:13:19,069 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8414b7acca30402d95ecc0b7edd21c5d, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397416 2024-12-03T06:13:19,069 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0854e18e65b143339e39c28c5761c177, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733206395866 2024-12-03T06:13:19,070 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cec7f3867f4a4846ab35e0a0cfe7933c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733206396211 2024-12-03T06:13:19,070 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 582123bae957447388d22835e4cdf2a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397430 2024-12-03T06:13:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T06:13:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:19,078 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,087 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#59 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:19,087 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/37779049d06e45809a0096a42cad0015 is 50, key is test_row_0/A:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:19,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/f8ad879d9d184d9ab72ea57e70fdbfc3 is 50, key is test_row_0/A:col10/1733206397792/Put/seqid=0 2024-12-03T06:13:19,089 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#61 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:19,090 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/9136b9d6117247409cdfad145266029f is 50, key is test_row_0/B:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741896_1072 (size=12983) 2024-12-03T06:13:19,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741897_1073 (size=12301) 2024-12-03T06:13:19,123 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/f8ad879d9d184d9ab72ea57e70fdbfc3 2024-12-03T06:13:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741898_1074 (size=12983) 2024-12-03T06:13:19,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/68bb6b3104984925af784ba2f97a6460 is 50, key is test_row_0/B:col10/1733206397792/Put/seqid=0 2024-12-03T06:13:19,145 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/9136b9d6117247409cdfad145266029f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/9136b9d6117247409cdfad145266029f 2024-12-03T06:13:19,155 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 9136b9d6117247409cdfad145266029f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:19,155 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:19,155 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=12, startTime=1733206399067; duration=0sec 2024-12-03T06:13:19,155 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:19,156 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:19,156 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T06:13:19,163 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73500 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T06:13:19,164 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:19,164 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:19,164 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6c23a1a1f18e4fddb05f18775b70d4b5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=71.8 K 2024-12-03T06:13:19,165 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c23a1a1f18e4fddb05f18775b70d4b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206394192 2024-12-03T06:13:19,165 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting edc2bfd7e4bd481b82b7381198630d71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733206394543 2024-12-03T06:13:19,166 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0904fce9405f45afb4d7c42cbff3337d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733206395843 2024-12-03T06:13:19,167 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1535663b3abf4c01a7c968b03d03cdef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733206395866 2024-12-03T06:13:19,167 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ad80fcd55ba4426a90f7b6316769c79e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733206396211 2024-12-03T06:13:19,168 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e95d020d4b146e78a6865be805c8e5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397430 2024-12-03T06:13:19,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741899_1075 (size=12301) 2024-12-03T06:13:19,178 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/68bb6b3104984925af784ba2f97a6460 2024-12-03T06:13:19,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6f27f3d4f7934103999a4890f2049622 is 50, key is test_row_0/C:col10/1733206397792/Put/seqid=0 2024-12-03T06:13:19,202 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:19,204 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/03bdaf6d09f84c99a1cb6994d39aae5a is 50, key is test_row_0/C:col10/1733206397761/Put/seqid=0 2024-12-03T06:13:19,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741900_1076 (size=12949) 2024-12-03T06:13:19,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741901_1077 (size=12301) 2024-12-03T06:13:19,241 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6f27f3d4f7934103999a4890f2049622 2024-12-03T06:13:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/f8ad879d9d184d9ab72ea57e70fdbfc3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3 2024-12-03T06:13:19,258 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:13:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/68bb6b3104984925af784ba2f97a6460 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460 2024-12-03T06:13:19,277 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:13:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6f27f3d4f7934103999a4890f2049622 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622 2024-12-03T06:13:19,302 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:13:19,304 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 8ec9e85614363b6e4e059ef6575a1df6 in 226ms, sequenceid=281, compaction requested=false 2024-12-03T06:13:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-03T06:13:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-03T06:13:19,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T06:13:19,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1560 sec 2024-12-03T06:13:19,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1640 sec 2024-12-03T06:13:19,518 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/37779049d06e45809a0096a42cad0015 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/37779049d06e45809a0096a42cad0015 2024-12-03T06:13:19,527 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 37779049d06e45809a0096a42cad0015(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:19,527 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:19,527 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=12, startTime=1733206399065; duration=0sec 2024-12-03T06:13:19,529 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:19,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:19,639 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/03bdaf6d09f84c99a1cb6994d39aae5a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/03bdaf6d09f84c99a1cb6994d39aae5a 2024-12-03T06:13:19,652 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 03bdaf6d09f84c99a1cb6994d39aae5a(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:19,652 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:19,652 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=10, startTime=1733206399067; duration=0sec 2024-12-03T06:13:19,652 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:19,652 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:19,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:13:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:19,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a5c22eb9bc094ed4a267f594fb6c37ca is 50, key is test_row_0/A:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741902_1078 (size=9857) 2024-12-03T06:13:19,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a5c22eb9bc094ed4a267f594fb6c37ca 2024-12-03T06:13:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206459997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206459999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206460000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206460001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a3e3f7dd3466479d9b2bf9685cde7d02 is 50, key is test_row_0/B:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:20,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206460004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741903_1079 (size=9857) 2024-12-03T06:13:20,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a3e3f7dd3466479d9b2bf9685cde7d02 2024-12-03T06:13:20,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/930fb393e75545ad826d4871886ff035 is 50, key is test_row_0/C:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:20,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741904_1080 (size=9857) 2024-12-03T06:13:20,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/930fb393e75545ad826d4871886ff035 2024-12-03T06:13:20,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a5c22eb9bc094ed4a267f594fb6c37ca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca 2024-12-03T06:13:20,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca, entries=100, sequenceid=296, filesize=9.6 K 2024-12-03T06:13:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a3e3f7dd3466479d9b2bf9685cde7d02 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02 2024-12-03T06:13:20,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02, entries=100, sequenceid=296, filesize=9.6 K 2024-12-03T06:13:20,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/930fb393e75545ad826d4871886ff035 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035 2024-12-03T06:13:20,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035, entries=100, sequenceid=296, filesize=9.6 K 2024-12-03T06:13:20,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8ec9e85614363b6e4e059ef6575a1df6 in 157ms, sequenceid=296, compaction requested=true 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:20,116 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:20,116 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:20,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:20,118 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:20,118 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:20,119 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:20,119 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:20,119 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,119 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,119 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/9136b9d6117247409cdfad145266029f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=34.3 K 2024-12-03T06:13:20,119 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/37779049d06e45809a0096a42cad0015, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=34.3 K 2024-12-03T06:13:20,121 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9136b9d6117247409cdfad145266029f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397430 2024-12-03T06:13:20,121 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37779049d06e45809a0096a42cad0015, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397430 2024-12-03T06:13:20,122 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 68bb6b3104984925af784ba2f97a6460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206397785 2024-12-03T06:13:20,122 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8ad879d9d184d9ab72ea57e70fdbfc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206397785 2024-12-03T06:13:20,122 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e3f7dd3466479d9b2bf9685cde7d02, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206399954 2024-12-03T06:13:20,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206399989, max timestamp=1733206400116Num uniques 859; 2024-12-03T06:13:20,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206399989, max timestamp=1733206400116Num uniques 859; 2024-12-03T06:13:20,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:20,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-12-03T06:13:20,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:20,132 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c22eb9bc094ed4a267f594fb6c37ca, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206399954 2024-12-03T06:13:20,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206399989, max timestamp=1733206400116Num uniques 859; 2024-12-03T06:13:20,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:20,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/db2f8a4bd3bd4394baa37167ad540899 is 50, key is test_row_0/A:col10/1733206400118/Put/seqid=0 2024-12-03T06:13:20,150 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:20,152 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/f447eff9f96141089503778fa52392a2 is 50, key is test_row_0/B:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:20,156 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:20,157 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/bb51edefd85b454a86280b98e7a4d457 is 50, key is test_row_0/A:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:20,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741906_1082 (size=13085) 2024-12-03T06:13:20,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741905_1081 (size=12301) 2024-12-03T06:13:20,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741907_1083 (size=13085) 2024-12-03T06:13:20,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206400174, max timestamp=1733206400180Num uniques 859; 2024-12-03T06:13:20,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206400174, max timestamp=1733206400180Num uniques 859; 2024-12-03T06:13:20,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206400174, max timestamp=1733206400184Num uniques 859; 2024-12-03T06:13:20,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:20,206 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/bb51edefd85b454a86280b98e7a4d457 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bb51edefd85b454a86280b98e7a4d457 2024-12-03T06:13:20,217 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into bb51edefd85b454a86280b98e7a4d457(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:20,217 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:20,217 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206400116; duration=0sec 2024-12-03T06:13:20,217 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:20,217 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:20,217 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:20,220 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:20,220 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:20,220 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,221 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/03bdaf6d09f84c99a1cb6994d39aae5a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=34.3 K 2024-12-03T06:13:20,222 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03bdaf6d09f84c99a1cb6994d39aae5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206397430 2024-12-03T06:13:20,224 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f27f3d4f7934103999a4890f2049622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206397785 2024-12-03T06:13:20,224 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930fb393e75545ad826d4871886ff035, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206399954 2024-12-03T06:13:20,238 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:20,238 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/253ae8e687b94091beafcfc85a69933c is 50, key is test_row_0/C:col10/1733206399958/Put/seqid=0 2024-12-03T06:13:20,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206400225, max timestamp=1733206400235Num uniques 1018; 2024-12-03T06:13:20,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206400225, max timestamp=1733206400239Num uniques 1018; 2024-12-03T06:13:20,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:13:20,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:13:20,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206400225, max timestamp=1733206400235Num uniques 1018; 2024-12-03T06:13:20,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:13:20,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741908_1084 (size=13051) 2024-12-03T06:13:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733206400263, max timestamp=1733206400280Num uniques 1027; 2024-12-03T06:13:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733206400263, max timestamp=1733206400280Num uniques 1027; 2024-12-03T06:13:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T06:13:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T06:13:20,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733206400263, max timestamp=1733206400280Num uniques 1027; 2024-12-03T06:13:20,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T06:13:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T06:13:20,300 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:20,301 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-03T06:13:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:20,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:20,306 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:20,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206460327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733206400309, max timestamp=1733206400334Num uniques 1136; 2024-12-03T06:13:20,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733206400309, max timestamp=1733206400334Num uniques 1136; 2024-12-03T06:13:20,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-03T06:13:20,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-03T06:13:20,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733206400309, max timestamp=1733206400334Num uniques 1136; 2024-12-03T06:13:20,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-12-03T06:13:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206460410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206460420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733206400347, max timestamp=1733206400419Num uniques 1195; 2024-12-03T06:13:20,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-03T06:13:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733206400347, max timestamp=1733206400419Num uniques 1195; 2024-12-03T06:13:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-03T06:13:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733206400347, max timestamp=1733206400419Num uniques 1195; 2024-12-03T06:13:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-03T06:13:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206460483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206460486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206460486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206460530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206460530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/db2f8a4bd3bd4394baa37167ad540899 2024-12-03T06:13:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206460590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,599 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/f447eff9f96141089503778fa52392a2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f447eff9f96141089503778fa52392a2 2024-12-03T06:13:20,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1e1723436c714170b2375bde5186e995 is 50, key is test_row_0/B:col10/1733206400118/Put/seqid=0 2024-12-03T06:13:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206460600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206460600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,614 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into f447eff9f96141089503778fa52392a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:20,614 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:20,614 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206400116; duration=0sec 2024-12-03T06:13:20,615 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:20,615 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:20,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741909_1085 (size=12301) 2024-12-03T06:13:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:20,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1e1723436c714170b2375bde5186e995 2024-12-03T06:13:20,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:20,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:20,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/073e6fef6a334c069191b3c0452eddb8 is 50, key is test_row_0/C:col10/1733206400118/Put/seqid=0 2024-12-03T06:13:20,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741910_1086 (size=12301) 2024-12-03T06:13:20,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/073e6fef6a334c069191b3c0452eddb8 2024-12-03T06:13:20,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/db2f8a4bd3bd4394baa37167ad540899 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899 2024-12-03T06:13:20,672 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/253ae8e687b94091beafcfc85a69933c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/253ae8e687b94091beafcfc85a69933c 2024-12-03T06:13:20,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T06:13:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1e1723436c714170b2375bde5186e995 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995 2024-12-03T06:13:20,683 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 253ae8e687b94091beafcfc85a69933c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:20,683 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:20,683 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206400116; duration=0sec 2024-12-03T06:13:20,684 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:20,684 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:20,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T06:13:20,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/073e6fef6a334c069191b3c0452eddb8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8 2024-12-03T06:13:20,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T06:13:20,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for 8ec9e85614363b6e4e059ef6575a1df6 in 569ms, sequenceid=318, compaction requested=false 2024-12-03T06:13:20,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:20,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=174.43 KB heapSize=487.78 KB 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:20,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:20,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e2c0e88f53b141b8b0333b293d62510a is 50, key is test_row_0/A:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:20,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206460753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206460755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741911_1087 (size=12301) 2024-12-03T06:13:20,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e2c0e88f53b141b8b0333b293d62510a 2024-12-03T06:13:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a0a4f7f121d64ad58a006c9873518ffe is 50, key is test_row_0/B:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:20,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741912_1088 (size=12301) 2024-12-03T06:13:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206460800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206460809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206460810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206460869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206460870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:20,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:20,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206461080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206461080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206461109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206461120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206461120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a0a4f7f121d64ad58a006c9873518ffe 2024-12-03T06:13:21,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/e0a74f22be90455686e3432353ca094c is 50, key is test_row_0/C:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:21,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741913_1089 (size=12301) 2024-12-03T06:13:21,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:21,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206461390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206461390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:21,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:21,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206461619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/e0a74f22be90455686e3432353ca094c 2024-12-03T06:13:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206461630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206461630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:21,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e2c0e88f53b141b8b0333b293d62510a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a 2024-12-03T06:13:21,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T06:13:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a0a4f7f121d64ad58a006c9873518ffe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe 2024-12-03T06:13:21,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T06:13:21,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/e0a74f22be90455686e3432353ca094c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c 2024-12-03T06:13:21,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T06:13:21,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~487.73 KB/499440, currentSize=13.42 KB/13740 for 8ec9e85614363b6e4e059ef6575a1df6 in 932ms, sequenceid=429, compaction requested=true 2024-12-03T06:13:21,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:21,679 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:21,680 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:21,680 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:21,681 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:21,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:21,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:21,681 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:21,681 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bb51edefd85b454a86280b98e7a4d457, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.8 K 2024-12-03T06:13:21,684 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb51edefd85b454a86280b98e7a4d457, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206397791 2024-12-03T06:13:21,685 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:21,685 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2f8a4bd3bd4394baa37167ad540899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733206400001 2024-12-03T06:13:21,685 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:21,685 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,685 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f447eff9f96141089503778fa52392a2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.8 K 2024-12-03T06:13:21,686 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f447eff9f96141089503778fa52392a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206397791 2024-12-03T06:13:21,686 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c0e88f53b141b8b0333b293d62510a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:21,687 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1723436c714170b2375bde5186e995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733206400001 2024-12-03T06:13:21,688 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a4f7f121d64ad58a006c9873518ffe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:21,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T06:13:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,705 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,711 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:21,712 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/55be89a4b368432481489ed6d25695b6 is 50, key is test_row_0/A:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:21,716 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:21,717 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a1dd7368175740859eab3ffdf577c87b is 50, key is test_row_0/B:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:21,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0e6ea36106da47768f749215af908c45 is 50, key is test_row_1/A:col10/1733206400751/Put/seqid=0 2024-12-03T06:13:21,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741915_1091 (size=13187) 2024-12-03T06:13:21,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741916_1092 (size=9857) 2024-12-03T06:13:21,739 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0e6ea36106da47768f749215af908c45 2024-12-03T06:13:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741914_1090 (size=13187) 2024-12-03T06:13:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d7e3b5fb019c44eb8f8527875cbf8e25 is 50, key is test_row_1/B:col10/1733206400751/Put/seqid=0 2024-12-03T06:13:21,757 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/55be89a4b368432481489ed6d25695b6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/55be89a4b368432481489ed6d25695b6 2024-12-03T06:13:21,768 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 55be89a4b368432481489ed6d25695b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:21,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:21,768 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206401678; duration=0sec 2024-12-03T06:13:21,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:21,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:21,770 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:21,771 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:21,772 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:21,772 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,772 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/253ae8e687b94091beafcfc85a69933c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.8 K 2024-12-03T06:13:21,772 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253ae8e687b94091beafcfc85a69933c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733206397791 2024-12-03T06:13:21,773 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073e6fef6a334c069191b3c0452eddb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733206400001 2024-12-03T06:13:21,773 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0a74f22be90455686e3432353ca094c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:21,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741917_1093 (size=9857) 2024-12-03T06:13:21,785 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d7e3b5fb019c44eb8f8527875cbf8e25 2024-12-03T06:13:21,788 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:21,789 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/4921f3f73a624d3ca4921439674f48a9 is 50, key is test_row_0/C:col10/1733206400481/Put/seqid=0 2024-12-03T06:13:21,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741918_1094 (size=13153) 2024-12-03T06:13:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8615117112e841609828b5f3a45f9c7c is 50, key is test_row_1/C:col10/1733206400751/Put/seqid=0 2024-12-03T06:13:21,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741919_1095 (size=9857) 2024-12-03T06:13:21,818 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8615117112e841609828b5f3a45f9c7c 2024-12-03T06:13:21,819 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/4921f3f73a624d3ca4921439674f48a9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/4921f3f73a624d3ca4921439674f48a9 2024-12-03T06:13:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/0e6ea36106da47768f749215af908c45 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45 2024-12-03T06:13:21,841 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 4921f3f73a624d3ca4921439674f48a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:21,841 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:21,841 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206401681; duration=0sec 2024-12-03T06:13:21,841 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:21,841 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:21,849 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45, entries=100, sequenceid=434, filesize=9.6 K 2024-12-03T06:13:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/d7e3b5fb019c44eb8f8527875cbf8e25 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25 2024-12-03T06:13:21,862 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25, entries=100, sequenceid=434, filesize=9.6 K 2024-12-03T06:13:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8615117112e841609828b5f3a45f9c7c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c 2024-12-03T06:13:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c, entries=100, sequenceid=434, filesize=9.6 K 2024-12-03T06:13:21,891 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8ec9e85614363b6e4e059ef6575a1df6 in 186ms, sequenceid=434, compaction requested=false 2024-12-03T06:13:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-03T06:13:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-03T06:13:21,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T06:13:21,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-12-03T06:13:21,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6010 sec 2024-12-03T06:13:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:21,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:13:21,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:21,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:21,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4f6b543aed014ea09f101622064e1f51 is 50, key is test_row_0/A:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:21,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741920_1096 (size=14741) 2024-12-03T06:13:21,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4f6b543aed014ea09f101622064e1f51 2024-12-03T06:13:21,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1c30317ffcc64a929c430b0f9da7caa0 is 50, key is test_row_0/B:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206401978, max timestamp=1733206401998Num uniques 859; 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206401978, max timestamp=1733206401998Num uniques 859; 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206401978, max timestamp=1733206401998Num uniques 859; 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:22,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741921_1097 (size=12301) 2024-12-03T06:13:22,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1c30317ffcc64a929c430b0f9da7caa0 2024-12-03T06:13:22,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2b8725e6d1074cc0a79015c21a8cbccd is 50, key is test_row_0/C:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741922_1098 (size=12301) 2024-12-03T06:13:22,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2b8725e6d1074cc0a79015c21a8cbccd 2024-12-03T06:13:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206462065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/4f6b543aed014ea09f101622064e1f51 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51 2024-12-03T06:13:22,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206462078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51, entries=200, sequenceid=447, filesize=14.4 K 2024-12-03T06:13:22,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/1c30317ffcc64a929c430b0f9da7caa0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0 2024-12-03T06:13:22,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T06:13:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/2b8725e6d1074cc0a79015c21a8cbccd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd 2024-12-03T06:13:22,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T06:13:22,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 8ec9e85614363b6e4e059ef6575a1df6 in 214ms, sequenceid=447, compaction requested=true 2024-12-03T06:13:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:22,136 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:13:22,138 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37785 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:22,139 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:22,139 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,139 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/55be89a4b368432481489ed6d25695b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=36.9 K 2024-12-03T06:13:22,145 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55be89a4b368432481489ed6d25695b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:22,146 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e6ea36106da47768f749215af908c45, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733206400751 2024-12-03T06:13:22,149 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6b543aed014ea09f101622064e1f51, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:22,151 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a1dd7368175740859eab3ffdf577c87b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a1dd7368175740859eab3ffdf577c87b 2024-12-03T06:13:22,164 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into a1dd7368175740859eab3ffdf577c87b(size=12.9 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:22,164 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,164 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206401680; duration=0sec 2024-12-03T06:13:22,165 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:13:22,165 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:22,165 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:22,166 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:22,170 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:22,170 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:22,170 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,171 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/4921f3f73a624d3ca4921439674f48a9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=34.5 K 2024-12-03T06:13:22,171 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4921f3f73a624d3ca4921439674f48a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:22,172 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8615117112e841609828b5f3a45f9c7c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733206400751 2024-12-03T06:13:22,172 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b8725e6d1074cc0a79015c21a8cbccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:22,177 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:22,178 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6b01b70132fd4db19f6b7ab4e1e52381 is 50, key is test_row_0/A:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:22,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:22,215 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#87 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:22,216 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/47d4960539b04af7bb45392af2baaec0 is 50, key is test_row_0/C:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206462218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/82b3cc1635a74c26bad85163abc91767 is 50, key is test_row_0/A:col10/1733206402062/Put/seqid=0 2024-12-03T06:13:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741923_1099 (size=13289) 2024-12-03T06:13:22,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206462221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,230 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/6b01b70132fd4db19f6b7ab4e1e52381 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6b01b70132fd4db19f6b7ab4e1e52381 2024-12-03T06:13:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,240 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 6b01b70132fd4db19f6b7ab4e1e52381(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:22,240 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,241 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206402136; duration=0sec 2024-12-03T06:13:22,241 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:22,241 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:22,241 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:22,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:22,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:22,243 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,244 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a1dd7368175740859eab3ffdf577c87b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=34.5 K 2024-12-03T06:13:22,244 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1dd7368175740859eab3ffdf577c87b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733206400470 2024-12-03T06:13:22,245 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e3b5fb019c44eb8f8527875cbf8e25, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733206400751 2024-12-03T06:13:22,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741924_1100 (size=13255) 2024-12-03T06:13:22,246 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c30317ffcc64a929c430b0f9da7caa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:22,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741925_1101 (size=14741) 2024-12-03T06:13:22,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/82b3cc1635a74c26bad85163abc91767 2024-12-03T06:13:22,272 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:22,272 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/6f3ffdf39de94475a84a120484e171af is 50, key is test_row_0/B:col10/1733206401919/Put/seqid=0 2024-12-03T06:13:22,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/766864656ede428a9a375b7ba67942bf is 50, key is test_row_0/B:col10/1733206402062/Put/seqid=0 2024-12-03T06:13:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741927_1103 (size=12301) 2024-12-03T06:13:22,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741926_1102 (size=13289) 2024-12-03T06:13:22,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/766864656ede428a9a375b7ba67942bf 2024-12-03T06:13:22,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/cada7dae08aa4d61b0061b9ec6183227 is 50, key is test_row_0/C:col10/1733206402062/Put/seqid=0 2024-12-03T06:13:22,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741928_1104 (size=12301) 2024-12-03T06:13:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206462329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206462340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T06:13:22,459 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:22,460 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-03T06:13:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:22,462 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:22,462 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:22,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:22,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206462540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206462549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:22,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206462639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,645 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206462650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206462652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,654 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,656 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,660 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/47d4960539b04af7bb45392af2baaec0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/47d4960539b04af7bb45392af2baaec0 2024-12-03T06:13:22,669 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 47d4960539b04af7bb45392af2baaec0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:22,672 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,672 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206402136; duration=0sec 2024-12-03T06:13:22,672 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:22,672 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:22,717 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/6f3ffdf39de94475a84a120484e171af as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6f3ffdf39de94475a84a120484e171af 2024-12-03T06:13:22,730 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 6f3ffdf39de94475a84a120484e171af(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:22,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/cada7dae08aa4d61b0061b9ec6183227 2024-12-03T06:13:22,731 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,731 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206402136; duration=0sec 2024-12-03T06:13:22,731 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:22,731 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/82b3cc1635a74c26bad85163abc91767 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767 2024-12-03T06:13:22,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767, entries=200, sequenceid=484, filesize=14.4 K 2024-12-03T06:13:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/766864656ede428a9a375b7ba67942bf as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf 2024-12-03T06:13:22,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf, entries=150, sequenceid=484, filesize=12.0 K 2024-12-03T06:13:22,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/cada7dae08aa4d61b0061b9ec6183227 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227 2024-12-03T06:13:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:22,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227, entries=150, sequenceid=484, filesize=12.0 K 2024-12-03T06:13:22,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 8ec9e85614363b6e4e059ef6575a1df6 in 593ms, sequenceid=484, compaction requested=false 2024-12-03T06:13:22,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:22,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:22,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a9a005448b174cf39d1f28e60debb864 is 50, key is test_row_0/A:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:22,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741929_1105 (size=14741) 2024-12-03T06:13:22,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206462904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206402887, max timestamp=1733206402905Num uniques 859; 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206402887, max timestamp=1733206402905Num uniques 859; 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206402887, max timestamp=1733206402905Num uniques 859; 2024-12-03T06:13:22,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:22,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:22,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:22,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206462987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206463019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206463099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206463229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a9a005448b174cf39d1f28e60debb864 2024-12-03T06:13:23,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b412461855544bfdb31d573e10d81cf3 is 50, key is test_row_0/B:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741930_1106 (size=12301) 2024-12-03T06:13:23,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b412461855544bfdb31d573e10d81cf3 2024-12-03T06:13:23,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/85e13f4be6d044d7b70299d80de8e589 is 50, key is test_row_0/C:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206463310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741931_1107 (size=12301) 2024-12-03T06:13:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206463540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206463620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:23,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:23,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/85e13f4be6d044d7b70299d80de8e589 2024-12-03T06:13:23,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/a9a005448b174cf39d1f28e60debb864 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864 2024-12-03T06:13:23,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864, entries=200, sequenceid=498, filesize=14.4 K 2024-12-03T06:13:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b412461855544bfdb31d573e10d81cf3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3 2024-12-03T06:13:23,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3, entries=150, sequenceid=498, filesize=12.0 K 2024-12-03T06:13:23,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/85e13f4be6d044d7b70299d80de8e589 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589 2024-12-03T06:13:23,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589, entries=150, sequenceid=498, filesize=12.0 K 2024-12-03T06:13:23,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8ec9e85614363b6e4e059ef6575a1df6 in 901ms, sequenceid=498, compaction requested=true 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:23,753 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:23,753 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:23,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:23,754 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:23,754 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:23,754 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:23,754 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:23,755 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,755 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,755 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6f3ffdf39de94475a84a120484e171af, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=37.0 K 2024-12-03T06:13:23,755 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6b01b70132fd4db19f6b7ab4e1e52381, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=41.8 K 2024-12-03T06:13:23,755 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3ffdf39de94475a84a120484e171af, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:23,755 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b01b70132fd4db19f6b7ab4e1e52381, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:23,755 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 766864656ede428a9a375b7ba67942bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733206402062 2024-12-03T06:13:23,755 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b3cc1635a74c26bad85163abc91767, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733206402062 2024-12-03T06:13:23,756 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b412461855544bfdb31d573e10d81cf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:23,756 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a005448b174cf39d1f28e60debb864, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:23,768 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:23,769 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:23,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/c69e8d521f23448bb4a2c7fc41c62bee is 50, key is test_row_0/A:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:23,769 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0320baf3414d4377a909bc7457ab2f4a is 50, key is test_row_0/B:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741933_1109 (size=13391) 2024-12-03T06:13:23,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741932_1108 (size=13391) 2024-12-03T06:13:23,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T06:13:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:23,850 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T06:13:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/87eac013ef384783ab3e6dde7fd40ba7 is 50, key is test_row_0/A:col10/1733206402970/Put/seqid=0 2024-12-03T06:13:23,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741934_1110 (size=12301) 2024-12-03T06:13:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:24,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:24,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206464079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206464130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,191 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/c69e8d521f23448bb4a2c7fc41c62bee as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c69e8d521f23448bb4a2c7fc41c62bee 2024-12-03T06:13:24,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206464189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,192 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0320baf3414d4377a909bc7457ab2f4a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0320baf3414d4377a909bc7457ab2f4a 2024-12-03T06:13:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,199 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into c69e8d521f23448bb4a2c7fc41c62bee(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:24,199 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:24,199 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206403753; duration=0sec 2024-12-03T06:13:24,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:24,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:24,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:24,201 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 0320baf3414d4377a909bc7457ab2f4a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:24,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:24,201 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206403753; duration=0sec 2024-12-03T06:13:24,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:24,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:24,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:24,202 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:24,202 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:24,202 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/47d4960539b04af7bb45392af2baaec0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=37.0 K 2024-12-03T06:13:24,202 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d4960539b04af7bb45392af2baaec0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733206401914 2024-12-03T06:13:24,203 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting cada7dae08aa4d61b0061b9ec6183227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733206402062 2024-12-03T06:13:24,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e13f4be6d044d7b70299d80de8e589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:24,214 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:24,215 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/bdf208570d7a43fabfa1d9935b8b0de4 is 50, key is test_row_0/C:col10/1733206402851/Put/seqid=0 2024-12-03T06:13:24,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741935_1111 (size=13357) 2024-12-03T06:13:24,228 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/bdf208570d7a43fabfa1d9935b8b0de4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/bdf208570d7a43fabfa1d9935b8b0de4 2024-12-03T06:13:24,233 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into bdf208570d7a43fabfa1d9935b8b0de4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:24,233 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:24,233 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206403753; duration=0sec 2024-12-03T06:13:24,234 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:24,234 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:24,264 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/87eac013ef384783ab3e6dde7fd40ba7 2024-12-03T06:13:24,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/03a5b8a13f5a470c81182627f24667c7 is 50, key is test_row_0/B:col10/1733206402970/Put/seqid=0 2024-12-03T06:13:24,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741936_1112 (size=12301) 2024-12-03T06:13:24,289 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/03a5b8a13f5a470c81182627f24667c7 2024-12-03T06:13:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6a91a3963fa8439ba8e6ecdd441bb8aa is 50, key is test_row_0/C:col10/1733206402970/Put/seqid=0 2024-12-03T06:13:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741937_1113 (size=12301) 2024-12-03T06:13:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206464399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:24,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50836 deadline: 1733206464670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1733206464670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,672 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,673 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50858 deadline: 1733206464679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,685 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 8ec9e85614363b6e4e059ef6575a1df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,704 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6a91a3963fa8439ba8e6ecdd441bb8aa 2024-12-03T06:13:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/87eac013ef384783ab3e6dde7fd40ba7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7 2024-12-03T06:13:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206464710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:24,718 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T06:13:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/03a5b8a13f5a470c81182627f24667c7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7 2024-12-03T06:13:24,726 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T06:13:24,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6a91a3963fa8439ba8e6ecdd441bb8aa as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa 2024-12-03T06:13:24,733 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T06:13:24,735 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 8ec9e85614363b6e4e059ef6575a1df6 in 885ms, sequenceid=534, compaction requested=false 2024-12-03T06:13:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-03T06:13:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-03T06:13:24,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T06:13:24,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2730 sec 2024-12-03T06:13:24,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.2790 sec 2024-12-03T06:13:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:25,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:13:25,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:25,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:25,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/590f4f9abf5d4f61869eb742b09885b6 is 50, key is test_row_0/A:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:25,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741938_1114 (size=14741) 2024-12-03T06:13:25,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:25,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206465217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206465219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:25,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206465329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206465539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:25,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/590f4f9abf5d4f61869eb742b09885b6 2024-12-03T06:13:25,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/3b1b892b8ee449d383b5990883564c94 is 50, key is test_row_0/B:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:25,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741939_1115 (size=12301) 2024-12-03T06:13:25,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/3b1b892b8ee449d383b5990883564c94 2024-12-03T06:13:25,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/04e51763ee234f839225543321375da5 is 50, key is test_row_0/C:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:25,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741940_1116 (size=12301) 2024-12-03T06:13:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206465850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:25,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/04e51763ee234f839225543321375da5 2024-12-03T06:13:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/590f4f9abf5d4f61869eb742b09885b6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6 2024-12-03T06:13:26,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6, entries=200, sequenceid=549, filesize=14.4 K 2024-12-03T06:13:26,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/3b1b892b8ee449d383b5990883564c94 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94 2024-12-03T06:13:26,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T06:13:26,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/04e51763ee234f839225543321375da5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5 2024-12-03T06:13:26,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T06:13:26,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8ec9e85614363b6e4e059ef6575a1df6 in 874ms, sequenceid=549, compaction requested=true 2024-12-03T06:13:26,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:26,019 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:26,021 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:26,021 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:26,021 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,021 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c69e8d521f23448bb4a2c7fc41c62bee, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=39.5 K 2024-12-03T06:13:26,022 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69e8d521f23448bb4a2c7fc41c62bee, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:26,022 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87eac013ef384783ab3e6dde7fd40ba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733206402963 2024-12-03T06:13:26,022 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590f4f9abf5d4f61869eb742b09885b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404061 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:26,026 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:26,027 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:26,027 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:26,027 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,027 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0320baf3414d4377a909bc7457ab2f4a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=37.1 K 2024-12-03T06:13:26,028 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0320baf3414d4377a909bc7457ab2f4a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:26,028 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 03a5b8a13f5a470c81182627f24667c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733206402963 2024-12-03T06:13:26,028 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1b892b8ee449d383b5990883564c94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404069 2024-12-03T06:13:26,031 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#104 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:26,031 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/891d1788a2814b95b0d115107ba8d7e0 is 50, key is test_row_0/A:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:26,039 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:26,039 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/53678e50a7c3422ebff448febd9d310b is 50, key is test_row_0/B:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741942_1118 (size=13493) 2024-12-03T06:13:26,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741941_1117 (size=13493) 2024-12-03T06:13:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:26,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:26,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206405194, max timestamp=1733206406240Num uniques 859; 2024-12-03T06:13:26,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206405194, max timestamp=1733206406240Num uniques 859; 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:26,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/34b44f76e565493c93ee55b5e1be5e06 is 50, key is test_row_0/A:col10/1733206406240/Put/seqid=0 2024-12-03T06:13:26,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741943_1119 (size=12301) 2024-12-03T06:13:26,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/34b44f76e565493c93ee55b5e1be5e06 2024-12-03T06:13:26,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0fd94b2596f4c0fa51600ec51979109 is 50, key is test_row_0/B:col10/1733206406240/Put/seqid=0 2024-12-03T06:13:26,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741944_1120 (size=12301) 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206406291, max timestamp=1733206406339Num uniques 859; 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206406291, max timestamp=1733206406339Num uniques 859; 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206406291, max timestamp=1733206406339Num uniques 859; 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:26,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206466340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206466407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206466450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,454 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/53678e50a7c3422ebff448febd9d310b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/53678e50a7c3422ebff448febd9d310b 2024-12-03T06:13:26,457 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/891d1788a2814b95b0d115107ba8d7e0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/891d1788a2814b95b0d115107ba8d7e0 2024-12-03T06:13:26,461 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 53678e50a7c3422ebff448febd9d310b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:26,461 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:26,461 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=13, startTime=1733206406026; duration=0sec 2024-12-03T06:13:26,461 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:26,461 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:26,461 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:26,462 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:26,462 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:26,463 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,464 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/bdf208570d7a43fabfa1d9935b8b0de4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=37.1 K 2024-12-03T06:13:26,465 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into 891d1788a2814b95b0d115107ba8d7e0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:26,465 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:26,465 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=13, startTime=1733206406019; duration=0sec 2024-12-03T06:13:26,465 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf208570d7a43fabfa1d9935b8b0de4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733206402214 2024-12-03T06:13:26,465 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:26,465 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:26,466 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a91a3963fa8439ba8e6ecdd441bb8aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733206402963 2024-12-03T06:13:26,466 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e51763ee234f839225543321375da5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404069 2024-12-03T06:13:26,473 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#108 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:26,474 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0fa947a0a6a84fc49ad0c39d46495cae is 50, key is test_row_0/C:col10/1733206404077/Put/seqid=0 2024-12-03T06:13:26,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741945_1121 (size=13459) 2024-12-03T06:13:26,492 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0fa947a0a6a84fc49ad0c39d46495cae as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0fa947a0a6a84fc49ad0c39d46495cae 2024-12-03T06:13:26,506 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 0fa947a0a6a84fc49ad0c39d46495cae(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:26,507 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:26,507 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=13, startTime=1733206406026; duration=0sec 2024-12-03T06:13:26,507 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:26,507 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:26,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206466509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T06:13:26,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:26,631 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-03T06:13:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T06:13:26,636 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:26,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:26,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206466659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0fd94b2596f4c0fa51600ec51979109 2024-12-03T06:13:26,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6cd5affda38d47c191dc25cc8ce5a751 is 50, key is test_row_0/C:col10/1733206406240/Put/seqid=0 2024-12-03T06:13:26,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741946_1122 (size=12301) 2024-12-03T06:13:26,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6cd5affda38d47c191dc25cc8ce5a751 2024-12-03T06:13:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206466719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:26,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/34b44f76e565493c93ee55b5e1be5e06 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06 2024-12-03T06:13:26,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T06:13:26,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/b0fd94b2596f4c0fa51600ec51979109 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109 2024-12-03T06:13:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T06:13:26,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T06:13:26,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/6cd5affda38d47c191dc25cc8ce5a751 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751 2024-12-03T06:13:26,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-03T06:13:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:26,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T06:13:26,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 8ec9e85614363b6e4e059ef6575a1df6 in 566ms, sequenceid=570, compaction requested=false 2024-12-03T06:13:26,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:26,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-03T06:13:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:26,945 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T06:13:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b645013d88ea4fa4932b88c67bb39487 is 50, key is test_row_0/A:col10/1733206406406/Put/seqid=0 2024-12-03T06:13:26,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741947_1123 (size=12301) 2024-12-03T06:13:27,005 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b645013d88ea4fa4932b88c67bb39487 2024-12-03T06:13:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206467014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a13bb02ebddf4180901ee8d0d1226cfd is 50, key is test_row_0/B:col10/1733206406406/Put/seqid=0 2024-12-03T06:13:27,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206467029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741948_1124 (size=12301) 2024-12-03T06:13:27,040 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a13bb02ebddf4180901ee8d0d1226cfd 2024-12-03T06:13:27,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/d16b9e87480b42618e1b0c2ace995d68 is 50, key is test_row_0/C:col10/1733206406406/Put/seqid=0 2024-12-03T06:13:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741949_1125 (size=12301) 2024-12-03T06:13:27,077 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/d16b9e87480b42618e1b0c2ace995d68 2024-12-03T06:13:27,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/b645013d88ea4fa4932b88c67bb39487 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487 2024-12-03T06:13:27,091 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T06:13:27,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/a13bb02ebddf4180901ee8d0d1226cfd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd 2024-12-03T06:13:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T06:13:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/d16b9e87480b42618e1b0c2ace995d68 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68 2024-12-03T06:13:27,103 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T06:13:27,104 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 8ec9e85614363b6e4e059ef6575a1df6 in 158ms, sequenceid=607, compaction requested=true 2024-12-03T06:13:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-03T06:13:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-03T06:13:27,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T06:13:27,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 468 msec 2024-12-03T06:13:27,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 477 msec 2024-12-03T06:13:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:27,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:27,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e6953d5fb0a14561b0da42a8d238ea6b is 50, key is test_row_0/A:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:27,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741950_1126 (size=12297) 2024-12-03T06:13:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206467208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T06:13:27,259 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:27,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-12-03T06:13:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:27,261 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:27,262 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:27,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206467320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:27,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:27,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206467529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1733206467539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e6953d5fb0a14561b0da42a8d238ea6b 2024-12-03T06:13:27,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/aa1257d794b048a699534a3014fa29d6 is 50, key is test_row_0/B:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741951_1127 (size=9857) 2024-12-03T06:13:27,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50844 deadline: 1733206467839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:13:27,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:27,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:27,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:27,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:27,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/aa1257d794b048a699534a3014fa29d6 2024-12-03T06:13:28,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8133b288c5bf451298415d1607e08862 is 50, key is test_row_0/C:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:28,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741952_1128 (size=9857) 2024-12-03T06:13:28,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8133b288c5bf451298415d1607e08862 2024-12-03T06:13:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/e6953d5fb0a14561b0da42a8d238ea6b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b 2024-12-03T06:13:28,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:28,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b, entries=150, sequenceid=619, filesize=12.0 K 2024-12-03T06:13:28,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:28,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/aa1257d794b048a699534a3014fa29d6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6 2024-12-03T06:13:28,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6, entries=100, sequenceid=619, filesize=9.6 K 2024-12-03T06:13:28,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/8133b288c5bf451298415d1607e08862 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862 2024-12-03T06:13:28,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862, entries=100, sequenceid=619, filesize=9.6 K 2024-12-03T06:13:28,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8ec9e85614363b6e4e059ef6575a1df6 in 929ms, sequenceid=619, compaction requested=true 2024-12-03T06:13:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:28,063 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:28,063 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:28,065 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:28,065 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/A is initiating minor compaction (all files) 2024-12-03T06:13:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ec9e85614363b6e4e059ef6575a1df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:28,065 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/A in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:28,065 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/891d1788a2814b95b0d115107ba8d7e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=49.2 K 2024-12-03T06:13:28,070 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891d1788a2814b95b0d115107ba8d7e0, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404069 2024-12-03T06:13:28,071 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b44f76e565493c93ee55b5e1be5e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1733206405194 2024-12-03T06:13:28,071 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47952 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:28,071 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b645013d88ea4fa4932b88c67bb39487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733206406389 2024-12-03T06:13:28,071 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/B is initiating minor compaction (all files) 2024-12-03T06:13:28,072 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/B in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,072 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/53678e50a7c3422ebff448febd9d310b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=46.8 K 2024-12-03T06:13:28,072 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6953d5fb0a14561b0da42a8d238ea6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733206407006 2024-12-03T06:13:28,072 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 53678e50a7c3422ebff448febd9d310b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404069 2024-12-03T06:13:28,073 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fd94b2596f4c0fa51600ec51979109, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1733206405194 2024-12-03T06:13:28,073 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T06:13:28,073 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,073 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a13bb02ebddf4180901ee8d0d1226cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733206406389 2024-12-03T06:13:28,074 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1257d794b048a699534a3014fa29d6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733206407006 2024-12-03T06:13:28,075 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T06:13:28,075 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,080 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,080 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,081 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T06:13:28,081 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,081 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,081 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,083 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T06:13:28,083 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,083 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,083 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,084 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#A#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:28,084 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/c2737ba2f2894e1a969e52837a7907f2 is 50, key is test_row_0/A:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:28,086 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:28,087 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/14a5d5199c654fda8e6c3a42a5317175 is 50, key is test_row_0/B:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:28,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741953_1129 (size=13629) 2024-12-03T06:13:28,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741954_1130 (size=13629) 2024-12-03T06:13:28,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,191 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/378410a1abd54bfd82d9fdfc3c0c14cd is 50, key is test_row_0/A:col10/1733206407207/Put/seqid=0 2024-12-03T06:13:28,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741955_1131 (size=12301) 2024-12-03T06:13:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. as already flushing 2024-12-03T06:13:28,361 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T06:13:28,362 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,362 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,362 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:28,498 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/c2737ba2f2894e1a969e52837a7907f2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c2737ba2f2894e1a969e52837a7907f2 2024-12-03T06:13:28,498 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/14a5d5199c654fda8e6c3a42a5317175 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/14a5d5199c654fda8e6c3a42a5317175 2024-12-03T06:13:28,503 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/A of 8ec9e85614363b6e4e059ef6575a1df6 into c2737ba2f2894e1a969e52837a7907f2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:28,503 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/B of 8ec9e85614363b6e4e059ef6575a1df6 into 14a5d5199c654fda8e6c3a42a5317175(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:28,503 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/A, priority=12, startTime=1733206408063; duration=0sec 2024-12-03T06:13:28,503 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/B, priority=12, startTime=1733206408063; duration=0sec 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:A 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:B 2024-12-03T06:13:28,503 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:28,505 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47918 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:28,505 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8ec9e85614363b6e4e059ef6575a1df6/C is initiating minor compaction (all files) 2024-12-03T06:13:28,505 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ec9e85614363b6e4e059ef6575a1df6/C in TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:28,505 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0fa947a0a6a84fc49ad0c39d46495cae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp, totalSize=46.8 K 2024-12-03T06:13:28,505 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fa947a0a6a84fc49ad0c39d46495cae, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733206404069 2024-12-03T06:13:28,506 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd5affda38d47c191dc25cc8ce5a751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1733206405194 2024-12-03T06:13:28,506 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16b9e87480b42618e1b0c2ace995d68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733206406389 2024-12-03T06:13:28,506 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8133b288c5bf451298415d1607e08862, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733206407006 2024-12-03T06:13:28,513 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ec9e85614363b6e4e059ef6575a1df6#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:28,514 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0d30c36ab0c64ea888eccd0ca4febf3e is 50, key is test_row_0/C:col10/1733206407006/Put/seqid=0 2024-12-03T06:13:28,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741956_1132 (size=13595) 2024-12-03T06:13:28,562 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T06:13:28,562 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,562 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,562 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,603 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/378410a1abd54bfd82d9fdfc3c0c14cd 2024-12-03T06:13:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0e55d894bb9742d780b6836f4363f630 is 50, key is test_row_0/B:col10/1733206407207/Put/seqid=0 2024-12-03T06:13:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741957_1133 (size=12301) 2024-12-03T06:13:28,701 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T06:13:28,701 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,701 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,701 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,711 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T06:13:28,711 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,711 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,712 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,721 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T06:13:28,721 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:28,721 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,721 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:28,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:28,924 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/0d30c36ab0c64ea888eccd0ca4febf3e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0d30c36ab0c64ea888eccd0ca4febf3e 2024-12-03T06:13:28,931 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ec9e85614363b6e4e059ef6575a1df6/C of 8ec9e85614363b6e4e059ef6575a1df6 into 0d30c36ab0c64ea888eccd0ca4febf3e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:28,931 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:28,931 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., storeName=8ec9e85614363b6e4e059ef6575a1df6/C, priority=12, startTime=1733206408065; duration=0sec 2024-12-03T06:13:28,931 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:28,931 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ec9e85614363b6e4e059ef6575a1df6:C 2024-12-03T06:13:29,024 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0e55d894bb9742d780b6836f4363f630 2024-12-03T06:13:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/5dacedc40fe642be8284ad67fbd12f2a is 50, key is test_row_0/C:col10/1733206407207/Put/seqid=0 2024-12-03T06:13:29,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741958_1134 (size=12301) 2024-12-03T06:13:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:29,441 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/5dacedc40fe642be8284ad67fbd12f2a 2024-12-03T06:13:29,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/378410a1abd54bfd82d9fdfc3c0c14cd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/378410a1abd54bfd82d9fdfc3c0c14cd 2024-12-03T06:13:29,459 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/378410a1abd54bfd82d9fdfc3c0c14cd, entries=150, sequenceid=639, filesize=12.0 K 2024-12-03T06:13:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/0e55d894bb9742d780b6836f4363f630 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0e55d894bb9742d780b6836f4363f630 2024-12-03T06:13:29,465 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0e55d894bb9742d780b6836f4363f630, entries=150, sequenceid=639, filesize=12.0 K 2024-12-03T06:13:29,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/5dacedc40fe642be8284ad67fbd12f2a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/5dacedc40fe642be8284ad67fbd12f2a 2024-12-03T06:13:29,472 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/5dacedc40fe642be8284ad67fbd12f2a, entries=150, sequenceid=639, filesize=12.0 K 2024-12-03T06:13:29,473 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 8ec9e85614363b6e4e059ef6575a1df6 in 1282ms, sequenceid=639, compaction requested=false 2024-12-03T06:13:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 8ec9e85614363b6e4e059ef6575a1df6: 2024-12-03T06:13:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-03T06:13:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-03T06:13:29,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T06:13:29,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-12-03T06:13:29,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.2180 sec 2024-12-03T06:13:29,768 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:13:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T06:13:31,430 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:31,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 155 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3118 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3080 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1463 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4385 rows 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-12-03T06:13:31,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4353 rows 2024-12-03T06:13:31,431 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:13:31,432 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:13:31,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:31,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:31,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:31,444 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:13:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:31,453 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206411452"}]},"ts":"1733206411452"} 2024-12-03T06:13:31,455 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:13:31,455 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:13:31,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:13:31,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, UNASSIGN}] 2024-12-03T06:13:31,463 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, UNASSIGN 2024-12-03T06:13:31,464 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8ec9e85614363b6e4e059ef6575a1df6, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:31,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, UNASSIGN because future has completed 2024-12-03T06:13:31,467 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:13:31,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:31,625 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:31,625 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:13:31,626 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 8ec9e85614363b6e4e059ef6575a1df6, disabling compactions & flushes 2024-12-03T06:13:31,626 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:31,627 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:31,627 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. after waiting 0 ms 2024-12-03T06:13:31,627 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:31,627 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 8ec9e85614363b6e4e059ef6575a1df6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:13:31,627 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=A 2024-12-03T06:13:31,628 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:31,628 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=B 2024-12-03T06:13:31,628 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:31,628 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ec9e85614363b6e4e059ef6575a1df6, store=C 2024-12-03T06:13:31,628 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:31,636 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3bb0ba20b6c748b69e6be07a536c549b is 50, key is test_row_0/A:col10/1733206408710/Put/seqid=0 2024-12-03T06:13:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:31,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741959_1135 (size=9857) 2024-12-03T06:13:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:32,197 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3bb0ba20b6c748b69e6be07a536c549b 2024-12-03T06:13:32,212 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/ceca7a0405424aacbe652de472828d07 is 50, key is test_row_0/B:col10/1733206408710/Put/seqid=0 2024-12-03T06:13:32,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741960_1136 (size=9857) 2024-12-03T06:13:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:32,619 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/ceca7a0405424aacbe652de472828d07 2024-12-03T06:13:32,636 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/01c38337874d4423b7840b12b303195b is 50, key is test_row_0/C:col10/1733206408710/Put/seqid=0 2024-12-03T06:13:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741961_1137 (size=9857) 2024-12-03T06:13:33,040 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/01c38337874d4423b7840b12b303195b 2024-12-03T06:13:33,047 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/A/3bb0ba20b6c748b69e6be07a536c549b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3bb0ba20b6c748b69e6be07a536c549b 2024-12-03T06:13:33,054 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3bb0ba20b6c748b69e6be07a536c549b, entries=100, sequenceid=650, filesize=9.6 K 2024-12-03T06:13:33,055 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/B/ceca7a0405424aacbe652de472828d07 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/ceca7a0405424aacbe652de472828d07 2024-12-03T06:13:33,060 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/ceca7a0405424aacbe652de472828d07, entries=100, sequenceid=650, filesize=9.6 K 2024-12-03T06:13:33,060 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/.tmp/C/01c38337874d4423b7840b12b303195b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/01c38337874d4423b7840b12b303195b 2024-12-03T06:13:33,065 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/01c38337874d4423b7840b12b303195b, entries=100, sequenceid=650, filesize=9.6 K 2024-12-03T06:13:33,065 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8ec9e85614363b6e4e059ef6575a1df6 in 1438ms, sequenceid=650, compaction requested=true 2024-12-03T06:13:33,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6593eff2d2e041e097670d1e77ee9652, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bcfaba1bc3d848899bc9f72872d6df53, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/10de59982c9144229aee2e2eccdafc52, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b582add2ffcf4adca3cfc95521e090be, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a3bef26f0c1240758ae4e262073b2bb2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/37779049d06e45809a0096a42cad0015, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bb51edefd85b454a86280b98e7a4d457, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/55be89a4b368432481489ed6d25695b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6b01b70132fd4db19f6b7ab4e1e52381, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c69e8d521f23448bb4a2c7fc41c62bee, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/891d1788a2814b95b0d115107ba8d7e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b] to archive 2024-12-03T06:13:33,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:13:33,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/94da9c146d1b4fe49357e3b0517ea7af 2024-12-03T06:13:33,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/830ced3f013143f5b35d9b95f8b809bd 2024-12-03T06:13:33,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0bd8bfc1f81845debd3e08bdec2a1f6d 2024-12-03T06:13:33,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6593eff2d2e041e097670d1e77ee9652 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6593eff2d2e041e097670d1e77ee9652 2024-12-03T06:13:33,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/067373fd734b44e4aeea0984449728f6 2024-12-03T06:13:33,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bcfaba1bc3d848899bc9f72872d6df53 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bcfaba1bc3d848899bc9f72872d6df53 2024-12-03T06:13:33,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/64b0e7fbfa5b421fb15f109ca002dc51 2024-12-03T06:13:33,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/806a40e908094d64816be627f951effa 2024-12-03T06:13:33,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/10de59982c9144229aee2e2eccdafc52 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/10de59982c9144229aee2e2eccdafc52 2024-12-03T06:13:33,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b65fe407f81f4cb0a14d7b2cacc6d474 2024-12-03T06:13:33,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b4c0384da8fc49f9bd1758b2812e0686 2024-12-03T06:13:33,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3dbf8498206544f6a0f100037835c32a 2024-12-03T06:13:33,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b582add2ffcf4adca3cfc95521e090be to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b582add2ffcf4adca3cfc95521e090be 2024-12-03T06:13:33,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/36b1fe0753bb4272b1d29e63f6848df2 2024-12-03T06:13:33,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6bd7e6e2d0684d8cb56ec57a6be5055f 2024-12-03T06:13:33,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a3bef26f0c1240758ae4e262073b2bb2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a3bef26f0c1240758ae4e262073b2bb2 2024-12-03T06:13:33,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6cb7d93656234815be8cb3b8c535af47 2024-12-03T06:13:33,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4795e72b5b7747e1b1ea72b8cfae8db0 2024-12-03T06:13:33,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/1a514a1f225a464286b184099fa65c0e 2024-12-03T06:13:33,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/8414b7acca30402d95ecc0b7edd21c5d 2024-12-03T06:13:33,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/37779049d06e45809a0096a42cad0015 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/37779049d06e45809a0096a42cad0015 2024-12-03T06:13:33,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/f8ad879d9d184d9ab72ea57e70fdbfc3 2024-12-03T06:13:33,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bb51edefd85b454a86280b98e7a4d457 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/bb51edefd85b454a86280b98e7a4d457 2024-12-03T06:13:33,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a5c22eb9bc094ed4a267f594fb6c37ca 2024-12-03T06:13:33,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/db2f8a4bd3bd4394baa37167ad540899 2024-12-03T06:13:33,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/55be89a4b368432481489ed6d25695b6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/55be89a4b368432481489ed6d25695b6 2024-12-03T06:13:33,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e2c0e88f53b141b8b0333b293d62510a 2024-12-03T06:13:33,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/0e6ea36106da47768f749215af908c45 2024-12-03T06:13:33,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/4f6b543aed014ea09f101622064e1f51 2024-12-03T06:13:33,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6b01b70132fd4db19f6b7ab4e1e52381 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/6b01b70132fd4db19f6b7ab4e1e52381 2024-12-03T06:13:33,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/82b3cc1635a74c26bad85163abc91767 2024-12-03T06:13:33,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/a9a005448b174cf39d1f28e60debb864 2024-12-03T06:13:33,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c69e8d521f23448bb4a2c7fc41c62bee to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c69e8d521f23448bb4a2c7fc41c62bee 2024-12-03T06:13:33,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/87eac013ef384783ab3e6dde7fd40ba7 2024-12-03T06:13:33,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/590f4f9abf5d4f61869eb742b09885b6 2024-12-03T06:13:33,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/891d1788a2814b95b0d115107ba8d7e0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/891d1788a2814b95b0d115107ba8d7e0 2024-12-03T06:13:33,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/34b44f76e565493c93ee55b5e1be5e06 2024-12-03T06:13:33,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/b645013d88ea4fa4932b88c67bb39487 2024-12-03T06:13:33,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/e6953d5fb0a14561b0da42a8d238ea6b 2024-12-03T06:13:33,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f4d4fce017d64d5d865e1c9a556d1736, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/142389f539af4290a35e3928aa4ff565, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/474c69911e9f4240a39773284526c64d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/dea8aefe824f40c9b3b83095eb4bf9fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0ffde0dc2d864560a5ad0ec7aa5e9877, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/9136b9d6117247409cdfad145266029f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f447eff9f96141089503778fa52392a2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a1dd7368175740859eab3ffdf577c87b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6f3ffdf39de94475a84a120484e171af, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0320baf3414d4377a909bc7457ab2f4a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/53678e50a7c3422ebff448febd9d310b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6] to archive 2024-12-03T06:13:33,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:13:33,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/064cfab7404042f1a6a94fb028d7d533 2024-12-03T06:13:33,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/85fa72c33c6e48a0808548549cee9126 2024-12-03T06:13:33,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f4d4fce017d64d5d865e1c9a556d1736 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f4d4fce017d64d5d865e1c9a556d1736 2024-12-03T06:13:33,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fb09d05b29b9467e82caacde7c7953f4 2024-12-03T06:13:33,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6778a8e261e24fbb8059f5e2b46340ec 2024-12-03T06:13:33,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/142389f539af4290a35e3928aa4ff565 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/142389f539af4290a35e3928aa4ff565 2024-12-03T06:13:33,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d4bc7602014b4fa5bb84df1384686dcc 2024-12-03T06:13:33,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a9c90eff71b44c348b63981cafb5beb4 2024-12-03T06:13:33,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/474c69911e9f4240a39773284526c64d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/474c69911e9f4240a39773284526c64d 2024-12-03T06:13:33,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/eac98821147d402ea1b1a39cbc8c2669 2024-12-03T06:13:33,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cdf2bacc5a2c4305b16285c8338cf571 2024-12-03T06:13:33,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/fe55d5bcf2754e01bd04b648c523813a 2024-12-03T06:13:33,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/dea8aefe824f40c9b3b83095eb4bf9fe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/dea8aefe824f40c9b3b83095eb4bf9fe 2024-12-03T06:13:33,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cde5fb072a7245e891a99c8ab7f7c1a2 2024-12-03T06:13:33,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0f20cf2c233413ebc93d1904c4a643d 2024-12-03T06:13:33,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0ffde0dc2d864560a5ad0ec7aa5e9877 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0ffde0dc2d864560a5ad0ec7aa5e9877 2024-12-03T06:13:33,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/669511ff1f3a4808a333843853d6274e 2024-12-03T06:13:33,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0854e18e65b143339e39c28c5761c177 2024-12-03T06:13:33,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/cec7f3867f4a4846ab35e0a0cfe7933c 2024-12-03T06:13:33,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/9136b9d6117247409cdfad145266029f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/9136b9d6117247409cdfad145266029f 2024-12-03T06:13:33,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/582123bae957447388d22835e4cdf2a4 2024-12-03T06:13:33,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/68bb6b3104984925af784ba2f97a6460 2024-12-03T06:13:33,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f447eff9f96141089503778fa52392a2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/f447eff9f96141089503778fa52392a2 2024-12-03T06:13:33,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a3e3f7dd3466479d9b2bf9685cde7d02 2024-12-03T06:13:33,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1e1723436c714170b2375bde5186e995 2024-12-03T06:13:33,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a1dd7368175740859eab3ffdf577c87b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a1dd7368175740859eab3ffdf577c87b 2024-12-03T06:13:33,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a0a4f7f121d64ad58a006c9873518ffe 2024-12-03T06:13:33,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/d7e3b5fb019c44eb8f8527875cbf8e25 2024-12-03T06:13:33,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6f3ffdf39de94475a84a120484e171af to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/6f3ffdf39de94475a84a120484e171af 2024-12-03T06:13:33,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/1c30317ffcc64a929c430b0f9da7caa0 2024-12-03T06:13:33,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/766864656ede428a9a375b7ba67942bf 2024-12-03T06:13:33,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0320baf3414d4377a909bc7457ab2f4a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0320baf3414d4377a909bc7457ab2f4a 2024-12-03T06:13:33,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b412461855544bfdb31d573e10d81cf3 2024-12-03T06:13:33,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/03a5b8a13f5a470c81182627f24667c7 2024-12-03T06:13:33,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/53678e50a7c3422ebff448febd9d310b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/53678e50a7c3422ebff448febd9d310b 2024-12-03T06:13:33,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/3b1b892b8ee449d383b5990883564c94 2024-12-03T06:13:33,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/b0fd94b2596f4c0fa51600ec51979109 2024-12-03T06:13:33,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/a13bb02ebddf4180901ee8d0d1226cfd 2024-12-03T06:13:33,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/aa1257d794b048a699534a3014fa29d6 2024-12-03T06:13:33,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/853e760848ae48468477faf47d695fca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8212307d311b4e689af1b478c79da6d9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0591f79c972f47988ab7f365a91b9c89, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6c23a1a1f18e4fddb05f18775b70d4b5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/03bdaf6d09f84c99a1cb6994d39aae5a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/253ae8e687b94091beafcfc85a69933c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/4921f3f73a624d3ca4921439674f48a9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/47d4960539b04af7bb45392af2baaec0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/bdf208570d7a43fabfa1d9935b8b0de4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0fa947a0a6a84fc49ad0c39d46495cae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862] to archive 2024-12-03T06:13:33,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:13:33,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6bba559202f345db9680f5991ff83ec4 2024-12-03T06:13:33,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/25989e22e30c4aef83b8d9c204276128 2024-12-03T06:13:33,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/853e760848ae48468477faf47d695fca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/853e760848ae48468477faf47d695fca 2024-12-03T06:13:33,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/405fdabc3f624a9abc23666992c7ab5c 2024-12-03T06:13:33,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/375b0735afaf47b4942e5b7f0a8d0eba 2024-12-03T06:13:33,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8212307d311b4e689af1b478c79da6d9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8212307d311b4e689af1b478c79da6d9 2024-12-03T06:13:33,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/a67077dd959c4af29cb0b35da8e9e34c 2024-12-03T06:13:33,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/94639ab2f8e540389c72a712a25fd010 2024-12-03T06:13:33,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0591f79c972f47988ab7f365a91b9c89 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0591f79c972f47988ab7f365a91b9c89 2024-12-03T06:13:33,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1b99d31090d44e59b622e483725ba592 2024-12-03T06:13:33,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2c4aef6a6a3842c0b8e0a0bb65324768 2024-12-03T06:13:33,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1579ac4f959241478c2708043607a60b 2024-12-03T06:13:33,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6c23a1a1f18e4fddb05f18775b70d4b5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6c23a1a1f18e4fddb05f18775b70d4b5 2024-12-03T06:13:33,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ed8f07eebf7349e59b1ad7340b40d841 2024-12-03T06:13:33,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/edc2bfd7e4bd481b82b7381198630d71 2024-12-03T06:13:33,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0904fce9405f45afb4d7c42cbff3337d 2024-12-03T06:13:33,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/1535663b3abf4c01a7c968b03d03cdef 2024-12-03T06:13:33,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/ad80fcd55ba4426a90f7b6316769c79e 2024-12-03T06:13:33,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/03bdaf6d09f84c99a1cb6994d39aae5a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/03bdaf6d09f84c99a1cb6994d39aae5a 2024-12-03T06:13:33,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8e95d020d4b146e78a6865be805c8e5c 2024-12-03T06:13:33,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6f27f3d4f7934103999a4890f2049622 2024-12-03T06:13:33,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/253ae8e687b94091beafcfc85a69933c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/253ae8e687b94091beafcfc85a69933c 2024-12-03T06:13:33,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/930fb393e75545ad826d4871886ff035 2024-12-03T06:13:33,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/073e6fef6a334c069191b3c0452eddb8 2024-12-03T06:13:33,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/4921f3f73a624d3ca4921439674f48a9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/4921f3f73a624d3ca4921439674f48a9 2024-12-03T06:13:33,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/e0a74f22be90455686e3432353ca094c 2024-12-03T06:13:33,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8615117112e841609828b5f3a45f9c7c 2024-12-03T06:13:33,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/47d4960539b04af7bb45392af2baaec0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/47d4960539b04af7bb45392af2baaec0 2024-12-03T06:13:33,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/2b8725e6d1074cc0a79015c21a8cbccd 2024-12-03T06:13:33,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/cada7dae08aa4d61b0061b9ec6183227 2024-12-03T06:13:33,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/bdf208570d7a43fabfa1d9935b8b0de4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/bdf208570d7a43fabfa1d9935b8b0de4 2024-12-03T06:13:33,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/85e13f4be6d044d7b70299d80de8e589 2024-12-03T06:13:33,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6a91a3963fa8439ba8e6ecdd441bb8aa 2024-12-03T06:13:33,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0fa947a0a6a84fc49ad0c39d46495cae to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0fa947a0a6a84fc49ad0c39d46495cae 2024-12-03T06:13:33,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/04e51763ee234f839225543321375da5 2024-12-03T06:13:33,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/6cd5affda38d47c191dc25cc8ce5a751 2024-12-03T06:13:33,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/d16b9e87480b42618e1b0c2ace995d68 2024-12-03T06:13:33,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/8133b288c5bf451298415d1607e08862 2024-12-03T06:13:33,235 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/recovered.edits/653.seqid, newMaxSeqId=653, maxSeqId=1 2024-12-03T06:13:33,236 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. 2024-12-03T06:13:33,236 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 8ec9e85614363b6e4e059ef6575a1df6: Waiting for close lock at 1733206411626Running coprocessor pre-close hooks at 1733206411626Disabling compacts and flushes for region at 1733206411626Disabling writes for close at 1733206411627 (+1 ms)Obtaining lock to block concurrent updates at 1733206411627Preparing flush snapshotting stores in 8ec9e85614363b6e4e059ef6575a1df6 at 1733206411627Finished memstore snapshotting TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733206411628 (+1 ms)Flushing stores of TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6. at 1733206411629 (+1 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/A: creating writer at 1733206411629Flushing 8ec9e85614363b6e4e059ef6575a1df6/A: appending metadata at 1733206411635 (+6 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/A: closing flushed file at 1733206411635Flushing 8ec9e85614363b6e4e059ef6575a1df6/B: creating writer at 1733206412207 (+572 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/B: appending metadata at 1733206412211 (+4 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/B: closing flushed file at 1733206412212 (+1 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/C: creating writer at 1733206412633 (+421 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/C: appending metadata at 1733206412635 (+2 ms)Flushing 8ec9e85614363b6e4e059ef6575a1df6/C: closing flushed file at 1733206412635Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ef851ae: reopening flushed file at 1733206413046 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d7de0e1: reopening flushed file at 1733206413054 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@152c2871: reopening flushed file at 1733206413060 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8ec9e85614363b6e4e059ef6575a1df6 in 1438ms, sequenceid=650, compaction requested=true at 1733206413066 (+6 ms)Writing region close event to WAL at 1733206413231 (+165 ms)Running coprocessor post-close hooks at 1733206413236 (+5 ms)Closed at 1733206413236 2024-12-03T06:13:33,239 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:33,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8ec9e85614363b6e4e059ef6575a1df6, regionState=CLOSED 2024-12-03T06:13:33,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:13:33,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-03T06:13:33,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ec9e85614363b6e4e059ef6575a1df6, server=122cddc95ec8,35255,1733206381424 in 1.7770 sec 2024-12-03T06:13:33,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T06:13:33,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ec9e85614363b6e4e059ef6575a1df6, UNASSIGN in 1.7840 sec 2024-12-03T06:13:33,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T06:13:33,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7920 sec 2024-12-03T06:13:33,252 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206413252"}]},"ts":"1733206413252"} 2024-12-03T06:13:33,254 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:13:33,254 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:13:33,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8080 sec 2024-12-03T06:13:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T06:13:33,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:33,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:13:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,608 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T06:13:33,609 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,616 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:33,621 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/recovered.edits] 2024-12-03T06:13:33,624 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/378410a1abd54bfd82d9fdfc3c0c14cd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/378410a1abd54bfd82d9fdfc3c0c14cd 2024-12-03T06:13:33,625 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3bb0ba20b6c748b69e6be07a536c549b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/3bb0ba20b6c748b69e6be07a536c549b 2024-12-03T06:13:33,626 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c2737ba2f2894e1a969e52837a7907f2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/A/c2737ba2f2894e1a969e52837a7907f2 2024-12-03T06:13:33,628 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0e55d894bb9742d780b6836f4363f630 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/0e55d894bb9742d780b6836f4363f630 2024-12-03T06:13:33,630 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/14a5d5199c654fda8e6c3a42a5317175 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/14a5d5199c654fda8e6c3a42a5317175 2024-12-03T06:13:33,631 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/ceca7a0405424aacbe652de472828d07 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/B/ceca7a0405424aacbe652de472828d07 2024-12-03T06:13:33,635 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/01c38337874d4423b7840b12b303195b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/01c38337874d4423b7840b12b303195b 2024-12-03T06:13:33,636 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0d30c36ab0c64ea888eccd0ca4febf3e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/0d30c36ab0c64ea888eccd0ca4febf3e 2024-12-03T06:13:33,637 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/5dacedc40fe642be8284ad67fbd12f2a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/C/5dacedc40fe642be8284ad67fbd12f2a 2024-12-03T06:13:33,640 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/recovered.edits/653.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6/recovered.edits/653.seqid 2024-12-03T06:13:33,641 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8ec9e85614363b6e4e059ef6575a1df6 2024-12-03T06:13:33,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:13:33,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T06:13:33,661 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:13:33,696 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:13:33,697 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,698 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:13:33,698 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206413698"}]},"ts":"9223372036854775807"} 2024-12-03T06:13:33,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:13:33,703 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8ec9e85614363b6e4e059ef6575a1df6, NAME => 'TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:13:33,703 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:13:33,703 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206413703"}]},"ts":"9223372036854775807"} 2024-12-03T06:13:33,707 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:13:33,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-12-03T06:13:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T06:13:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:13:33,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:33,734 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;122cddc95ec8:35255-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-161939200_22 at /127.0.0.1:46282 [Waiting for operation #272] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1228833144_22 at /127.0.0.1:57366 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=442 (was 280) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12798 (was 12419) - AvailableMemoryMB LEAK? - 2024-12-03T06:13:33,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=442, ProcessCount=11, AvailableMemoryMB=12797 2024-12-03T06:13:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:13:33,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:13:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:33,749 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:13:33,749 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:33,749 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-12-03T06:13:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:33,750 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:13:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741962_1138 (size=1061) 2024-12-03T06:13:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:34,160 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:13:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741963_1139 (size=53) 2024-12-03T06:13:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:34,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:34,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 87b34dfe780d9655f141bfd2362b0642, disabling compactions & flushes 2024-12-03T06:13:34,567 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. after waiting 0 ms 2024-12-03T06:13:34,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,567 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87b34dfe780d9655f141bfd2362b0642: Waiting for close lock at 1733206414567Disabling compacts and flushes for region at 1733206414567Disabling writes for close at 1733206414567Writing region close event to WAL at 1733206414567Closed at 1733206414567 2024-12-03T06:13:34,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:13:34,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206414569"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206414569"}]},"ts":"1733206414569"} 2024-12-03T06:13:34,571 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:13:34,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:13:34,573 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206414573"}]},"ts":"1733206414573"} 2024-12-03T06:13:34,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:13:34,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, ASSIGN}] 2024-12-03T06:13:34,577 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, ASSIGN 2024-12-03T06:13:34,578 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:13:34,729 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:34,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, ASSIGN because future has completed 2024-12-03T06:13:34,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:34,888 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,888 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:13:34,889 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,889 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:34,889 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,889 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:34,891 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,892 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:34,893 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName A 2024-12-03T06:13:34,893 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:34,893 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:34,894 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,895 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:34,895 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName B 2024-12-03T06:13:34,895 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:34,896 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:34,896 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,897 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:34,897 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName C 2024-12-03T06:13:34,897 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:34,898 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:34,898 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,898 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,899 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,900 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,901 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,901 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,901 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:13:34,903 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,905 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:13:34,905 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 87b34dfe780d9655f141bfd2362b0642; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66728977, jitterRate=-0.005660757422447205}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:13:34,905 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:34,906 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 87b34dfe780d9655f141bfd2362b0642: Running coprocessor pre-open hook at 1733206414889Writing region info on filesystem at 1733206414889Initializing all the Stores at 1733206414890 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206414890Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206414890Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206414890Cleaning up temporary data from old regions at 1733206414901 (+11 ms)Running coprocessor post-open hooks at 1733206414905 (+4 ms)Region opened successfully at 1733206414906 (+1 ms) 2024-12-03T06:13:34,907 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., pid=38, masterSystemTime=1733206414884 2024-12-03T06:13:34,909 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,909 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:34,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:34,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:13:34,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-03T06:13:34,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 in 180 msec 2024-12-03T06:13:34,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T06:13:34,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, ASSIGN in 338 msec 2024-12-03T06:13:34,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:13:34,921 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206414921"}]},"ts":"1733206414921"} 2024-12-03T06:13:34,923 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:13:34,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:13:34,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-12-03T06:13:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T06:13:35,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:35,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:35,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:35,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:35,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61274f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:35,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf8b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:35,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:35,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:35,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:13:35,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:13:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ad97762 2024-12-03T06:13:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:13:35,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:13:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:13:35,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:13:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:35,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741964_1140 (size=1097) 2024-12-03T06:13:36,358 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T06:13:36,358 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T06:13:36,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:13:36,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE}] 2024-12-03T06:13:36,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE 2024-12-03T06:13:36,375 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:36,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE because future has completed 2024-12-03T06:13:36,378 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:13:36,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:36,531 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,531 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:13:36,532 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 87b34dfe780d9655f141bfd2362b0642, disabling compactions & flushes 2024-12-03T06:13:36,532 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,532 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,532 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. after waiting 0 ms 2024-12-03T06:13:36,532 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,537 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:13:36,538 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,538 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 87b34dfe780d9655f141bfd2362b0642: Waiting for close lock at 1733206416532Running coprocessor pre-close hooks at 1733206416532Disabling compacts and flushes for region at 1733206416532Disabling writes for close at 1733206416532Writing region close event to WAL at 1733206416532Running coprocessor post-close hooks at 1733206416538 (+6 ms)Closed at 1733206416538 2024-12-03T06:13:36,538 WARN [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 87b34dfe780d9655f141bfd2362b0642 to self. 2024-12-03T06:13:36,541 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=CLOSED 2024-12-03T06:13:36,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:13:36,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-12-03T06:13:36,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 in 167 msec 2024-12-03T06:13:36,548 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE; state=CLOSED, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=true 2024-12-03T06:13:36,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:36,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE because future has completed 2024-12-03T06:13:36,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:36,859 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,859 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:13:36,859 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,860 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:13:36,860 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,860 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,866 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,867 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:36,874 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName A 2024-12-03T06:13:36,876 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:36,877 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:36,878 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,879 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:36,879 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName B 2024-12-03T06:13:36,879 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:36,880 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:36,880 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,880 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:13:36,880 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b34dfe780d9655f141bfd2362b0642 columnFamilyName C 2024-12-03T06:13:36,881 DEBUG [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:36,881 INFO [StoreOpener-87b34dfe780d9655f141bfd2362b0642-1 {}] regionserver.HStore(327): Store=87b34dfe780d9655f141bfd2362b0642/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:13:36,881 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,881 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,882 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,883 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,884 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,884 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,885 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:13:36,886 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,887 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 87b34dfe780d9655f141bfd2362b0642; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74340928, jitterRate=0.10776615142822266}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:13:36,887 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,887 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 87b34dfe780d9655f141bfd2362b0642: Running coprocessor pre-open hook at 1733206416860Writing region info on filesystem at 1733206416860Initializing all the Stores at 1733206416861 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206416861Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206416862 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206416862Cleaning up temporary data from old regions at 1733206416884 (+22 ms)Running coprocessor post-open hooks at 1733206416887 (+3 ms)Region opened successfully at 1733206416887 2024-12-03T06:13:36,888 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., pid=43, masterSystemTime=1733206416854 2024-12-03T06:13:36,891 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,891 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:36,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=OPEN, openSeqNum=5, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:36,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:13:36,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-12-03T06:13:36,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 in 192 msec 2024-12-03T06:13:36,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T06:13:36,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, REOPEN/MOVE in 524 msec 2024-12-03T06:13:36,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-03T06:13:36,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-12-03T06:13:36,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 981 msec 2024-12-03T06:13:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-03T06:13:36,909 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:36,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f05ff6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0451e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8abdfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755e65cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4c9f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b84d9e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11259674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0f22ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,942 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c8e601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4becd7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a19d68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348e8dfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cfb6c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,952 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bfc8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,954 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba5a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679bff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:13:36,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:13:36,958 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:13:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:13:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:13:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb1d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:13:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:13:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:36,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:13:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:13:36,964 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,964 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,964 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,964 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:36,964 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,964 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,965 DEBUG [Thread-691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T06:13:36,966 DEBUG [Thread-708 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,966 DEBUG [Thread-708 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:36,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,969 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,970 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,971 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:13:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:13:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:13:36,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:36,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:13:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:13:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:36,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:37,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b779313a43a04f538cf074a3242a6336_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_1/A:col10/1733206416979/Put/seqid=0 2024-12-03T06:13:37,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741965_1141 (size=17034) 2024-12-03T06:13:37,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733206417028, max timestamp=1733206417047Num uniques 859; 2024-12-03T06:13:37,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733206417028, max timestamp=1733206417047Num uniques 859; 2024-12-03T06:13:37,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T06:13:37,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T06:13:37,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733206417028, max timestamp=1733206417047Num uniques 859; 2024-12-03T06:13:37,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T06:13:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:37,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206477076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206477087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206477088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206477093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206477094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206477202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206477202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206477201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206477202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206477210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:37,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206477410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206477410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206477410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206477410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206477420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,436 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:37,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,447 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b779313a43a04f538cf074a3242a6336_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b779313a43a04f538cf074a3242a6336_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:37,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e1e503c2d9de4f0a8acef50b00a0fed5, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:37,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e1e503c2d9de4f0a8acef50b00a0fed5 is 175, key is test_row_1/A:col10/1733206416979/Put/seqid=0 2024-12-03T06:13:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741966_1142 (size=48135) 2024-12-03T06:13:37,472 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e1e503c2d9de4f0a8acef50b00a0fed5 2024-12-03T06:13:37,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b63ea0cd1aa44d959287f0f65b295a14 is 50, key is test_row_1/B:col10/1733206416979/Put/seqid=0 2024-12-03T06:13:37,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741967_1143 (size=9657) 2024-12-03T06:13:37,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b63ea0cd1aa44d959287f0f65b295a14 2024-12-03T06:13:37,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/cb9f986e400744499ec607984d2035e9 is 50, key is test_row_1/C:col10/1733206416979/Put/seqid=0 2024-12-03T06:13:37,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741968_1144 (size=9657) 2024-12-03T06:13:37,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/cb9f986e400744499ec607984d2035e9 2024-12-03T06:13:37,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e1e503c2d9de4f0a8acef50b00a0fed5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5 2024-12-03T06:13:37,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5, entries=250, sequenceid=14, filesize=47.0 K 2024-12-03T06:13:37,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b63ea0cd1aa44d959287f0f65b295a14 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14 2024-12-03T06:13:37,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14, entries=100, sequenceid=14, filesize=9.4 K 2024-12-03T06:13:37,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/cb9f986e400744499ec607984d2035e9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9 2024-12-03T06:13:37,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9, entries=100, sequenceid=14, filesize=9.4 K 2024-12-03T06:13:37,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 87b34dfe780d9655f141bfd2362b0642 in 684ms, sequenceid=14, compaction requested=false 2024-12-03T06:13:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T06:13:37,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:37,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-03T06:13:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206477730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206477730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120319c9cde737a94c71baee0134730bb3be_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206417721/Put/seqid=0 2024-12-03T06:13:37,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741969_1145 (size=17034) 2024-12-03T06:13:37,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:37,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120319c9cde737a94c71baee0134730bb3be_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120319c9cde737a94c71baee0134730bb3be_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:37,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c5b5f7177d974e0bb134bdcb6f5fab9c, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:37,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c5b5f7177d974e0bb134bdcb6f5fab9c is 175, key is test_row_0/A:col10/1733206417721/Put/seqid=0 2024-12-03T06:13:37,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741970_1146 (size=48139) 2024-12-03T06:13:37,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c5b5f7177d974e0bb134bdcb6f5fab9c 2024-12-03T06:13:37,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/40966d3ad1af473bb278959628ad1e46 is 50, key is test_row_0/B:col10/1733206417721/Put/seqid=0 2024-12-03T06:13:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206477840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206477850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206477850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206477851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741971_1147 (size=12001) 2024-12-03T06:13:37,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/40966d3ad1af473bb278959628ad1e46 2024-12-03T06:13:37,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/67f9e500ec0549e3bf318b2f462c86cc is 50, key is test_row_0/C:col10/1733206417721/Put/seqid=0 2024-12-03T06:13:37,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:37,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:37,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741972_1148 (size=12001) 2024-12-03T06:13:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206478050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:38,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206478060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206478060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206478060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:38,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:38,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:38,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206478242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/67f9e500ec0549e3bf318b2f462c86cc 2024-12-03T06:13:38,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c5b5f7177d974e0bb134bdcb6f5fab9c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c 2024-12-03T06:13:38,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c, entries=250, sequenceid=51, filesize=47.0 K 2024-12-03T06:13:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/40966d3ad1af473bb278959628ad1e46 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46 2024-12-03T06:13:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T06:13:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/67f9e500ec0549e3bf318b2f462c86cc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc 2024-12-03T06:13:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T06:13:38,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 87b34dfe780d9655f141bfd2362b0642 in 644ms, sequenceid=51, compaction requested=false 2024-12-03T06:13:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T06:13:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:38,377 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:13:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032082e353d3b54603aff07c190a8cee84_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_1/A:col10/1733206417731/Put/seqid=0 2024-12-03T06:13:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741973_1149 (size=9714) 2024-12-03T06:13:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,430 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032082e353d3b54603aff07c190a8cee84_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032082e353d3b54603aff07c190a8cee84_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/d23bd4a78e804485addf2819e76f81d1, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/d23bd4a78e804485addf2819e76f81d1 is 175, key is test_row_1/A:col10/1733206417731/Put/seqid=0 2024-12-03T06:13:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741974_1150 (size=22361) 2024-12-03T06:13:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,457 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/d23bd4a78e804485addf2819e76f81d1 2024-12-03T06:13:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:38,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/9ed38e18905b4094b61f4fe617b7fa91 is 50, key is test_row_1/B:col10/1733206417731/Put/seqid=0 2024-12-03T06:13:38,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206418491, max timestamp=1733206418503Num uniques 859; 2024-12-03T06:13:38,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206418491, max timestamp=1733206418503Num uniques 859; 2024-12-03T06:13:38,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:38,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:38,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206418491, max timestamp=1733206418503Num uniques 859; 2024-12-03T06:13:38,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:13:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741975_1151 (size=9657) 2024-12-03T06:13:38,514 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/9ed38e18905b4094b61f4fe617b7fa91 2024-12-03T06:13:38,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/3914e52030ab49afa029065d1d39c1a4 is 50, key is test_row_1/C:col10/1733206417731/Put/seqid=0 2024-12-03T06:13:38,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206478552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206478553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206478553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206478571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741976_1152 (size=9657) 2024-12-03T06:13:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206478670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206478670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206478670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206478679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,760 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:13:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206478890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206478890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206478890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206478890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:38,976 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/3914e52030ab49afa029065d1d39c1a4 2024-12-03T06:13:38,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/d23bd4a78e804485addf2819e76f81d1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1 2024-12-03T06:13:38,991 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1, entries=100, sequenceid=59, filesize=21.8 K 2024-12-03T06:13:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/9ed38e18905b4094b61f4fe617b7fa91 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91 2024-12-03T06:13:39,000 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91, entries=100, sequenceid=59, filesize=9.4 K 2024-12-03T06:13:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/3914e52030ab49afa029065d1d39c1a4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4 2024-12-03T06:13:39,012 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4, entries=100, sequenceid=59, filesize=9.4 K 2024-12-03T06:13:39,013 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 87b34dfe780d9655f141bfd2362b0642 in 636ms, sequenceid=59, compaction requested=true 2024-12-03T06:13:39,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T06:13:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T06:13:39,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T06:13:39,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0380 sec 2024-12-03T06:13:39,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0580 sec 2024-12-03T06:13:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T06:13:39,120 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:39,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T06:13:39,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:39,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:39,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:39,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=161.02 KB heapSize=456 KB 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:13:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206479210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206479216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206479217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bc6c85446cbd4de8a6e37e3e0ba9715a_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206479222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741977_1153 (size=12154) 2024-12-03T06:13:39,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:39,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bc6c85446cbd4de8a6e37e3e0ba9715a_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc6c85446cbd4de8a6e37e3e0ba9715a_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:39,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/8c51ca277ad04619b53d9982a8555b2c, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:39,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/8c51ca277ad04619b53d9982a8555b2c is 175, key is test_row_0/A:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:39,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206479260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,264 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 87b34dfe780d9655f141bfd2362b0642 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741978_1154 (size=30955) 2024-12-03T06:13:39,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206479319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206479329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206479340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206479529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206479540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206479550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:39,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:39,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/8c51ca277ad04619b53d9982a8555b2c 2024-12-03T06:13:39,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b445c67ae620472eb5c112950f324d7d is 50, key is test_row_0/B:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:39,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741979_1155 (size=12001) 2024-12-03T06:13:39,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b445c67ae620472eb5c112950f324d7d 2024-12-03T06:13:39,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a8586b9bf9d4418f87eb0b289c6f21f8 is 50, key is test_row_0/C:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741980_1156 (size=12001) 2024-12-03T06:13:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206479729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:39,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206479840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206479850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206479859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:39,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:39,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:39,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:39,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:40,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:40,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a8586b9bf9d4418f87eb0b289c6f21f8 2024-12-03T06:13:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/8c51ca277ad04619b53d9982a8555b2c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c 2024-12-03T06:13:40,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c, entries=150, sequenceid=98, filesize=30.2 K 2024-12-03T06:13:40,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/b445c67ae620472eb5c112950f324d7d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d 2024-12-03T06:13:40,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T06:13:40,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a8586b9bf9d4418f87eb0b289c6f21f8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8 2024-12-03T06:13:40,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T06:13:40,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~455.95 KB/466896, currentSize=20.13 KB/20610 for 87b34dfe780d9655f141bfd2362b0642 in 971ms, sequenceid=98, compaction requested=true 2024-12-03T06:13:40,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:40,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:40,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:40,175 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:40,175 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:40,177 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:40,177 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:40,177 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:40,177 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:40,178 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,178 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,178 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=146.1 K 2024-12-03T06:13:40,178 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=42.3 K 2024-12-03T06:13:40,178 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,178 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c] 2024-12-03T06:13:40,178 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b63ea0cd1aa44d959287f0f65b295a14, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733206416979 2024-12-03T06:13:40,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e503c2d9de4f0a8acef50b00a0fed5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733206416977 2024-12-03T06:13:40,179 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 40966d3ad1af473bb278959628ad1e46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206417074 2024-12-03T06:13:40,179 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed38e18905b4094b61f4fe617b7fa91, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206417731 2024-12-03T06:13:40,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5b5f7177d974e0bb134bdcb6f5fab9c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206417074 2024-12-03T06:13:40,180 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d23bd4a78e804485addf2819e76f81d1, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206417731 2024-12-03T06:13:40,180 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b445c67ae620472eb5c112950f324d7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:40,180 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c51ca277ad04619b53d9982a8555b2c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:40,192 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:40,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T06:13:40,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,212 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T06:13:40,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:40,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:40,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:40,215 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ab4bf185fc654e3c9a05f6beb1175e51_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:40,219 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ab4bf185fc654e3c9a05f6beb1175e51_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:40,219 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ab4bf185fc654e3c9a05f6beb1175e51_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:40,223 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#138 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:40,223 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/5ba730daf2864576b30187ce4856af16 is 50, key is test_row_0/B:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741981_1157 (size=4469) 2024-12-03T06:13:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120314b8c496fe9a41b0ad9c3d3372f54a5e_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206419215/Put/seqid=0 2024-12-03T06:13:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:40,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741983_1159 (size=12154) 2024-12-03T06:13:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741982_1158 (size=12139) 2024-12-03T06:13:40,289 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/5ba730daf2864576b30187ce4856af16 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/5ba730daf2864576b30187ce4856af16 2024-12-03T06:13:40,298 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 5ba730daf2864576b30187ce4856af16(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:40,298 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:40,298 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=12, startTime=1733206420175; duration=0sec 2024-12-03T06:13:40,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:40,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:40,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:40,301 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:40,301 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:40,301 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,301 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=42.3 K 2024-12-03T06:13:40,302 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cb9f986e400744499ec607984d2035e9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733206416979 2024-12-03T06:13:40,303 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f9e500ec0549e3bf318b2f462c86cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206417074 2024-12-03T06:13:40,303 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3914e52030ab49afa029065d1d39c1a4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206417731 2024-12-03T06:13:40,304 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a8586b9bf9d4418f87eb0b289c6f21f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:40,336 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:40,337 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/9e9ef0f6b1f642e2977d40b2c46d2f2a is 50, key is test_row_0/C:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741984_1160 (size=12139) 2024-12-03T06:13:40,357 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/9e9ef0f6b1f642e2977d40b2c46d2f2a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9e9ef0f6b1f642e2977d40b2c46d2f2a 2024-12-03T06:13:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:40,369 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into 9e9ef0f6b1f642e2977d40b2c46d2f2a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:40,369 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:40,369 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=12, startTime=1733206420176; duration=0sec 2024-12-03T06:13:40,369 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:40,369 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420386, max timestamp=1733206420492Num uniques 859; 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420386, max timestamp=1733206420492Num uniques 859; 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420386, max timestamp=1733206420492Num uniques 859; 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206420533, max timestamp=1733206420549Num uniques 1068; 2024-12-03T06:13:40,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T06:13:40,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206420533, max timestamp=1733206420549Num uniques 1068; 2024-12-03T06:13:40,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T06:13:40,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206420533, max timestamp=1733206420549Num uniques 1068; 2024-12-03T06:13:40,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T06:13:40,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206420585, max timestamp=1733206420607Num uniques 1027; 2024-12-03T06:13:40,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:13:40,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206420585, max timestamp=1733206420607Num uniques 1027; 2024-12-03T06:13:40,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:13:40,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206420585, max timestamp=1733206420607Num uniques 1027; 2024-12-03T06:13:40,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:13:40,632 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#137 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:40,632 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/df6afeff21984e05a16d61dd8ae5d2fe is 175, key is test_row_0/A:col10/1733206418551/Put/seqid=0 2024-12-03T06:13:40,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741985_1161 (size=31093) 2024-12-03T06:13:40,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733206420634, max timestamp=1733206420659Num uniques 1036; 2024-12-03T06:13:40,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-03T06:13:40,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733206420634, max timestamp=1733206420659Num uniques 1036; 2024-12-03T06:13:40,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-03T06:13:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733206420634, max timestamp=1733206420659Num uniques 1036; 2024-12-03T06:13:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-03T06:13:40,671 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/df6afeff21984e05a16d61dd8ae5d2fe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe 2024-12-03T06:13:40,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:40,688 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120314b8c496fe9a41b0ad9c3d3372f54a5e_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314b8c496fe9a41b0ad9c3d3372f54a5e_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b914ffc0e3c544a3aed7e9287570f44e, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:40,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b914ffc0e3c544a3aed7e9287570f44e is 175, key is test_row_0/A:col10/1733206419215/Put/seqid=0 2024-12-03T06:13:40,695 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into df6afeff21984e05a16d61dd8ae5d2fe(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:40,695 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:40,695 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=12, startTime=1733206420175; duration=0sec 2024-12-03T06:13:40,701 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:40,701 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:40,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=445, cellSize=20380, totalHeapSize=46520, min timestamp=1733206420669, max timestamp=1733206420712Num uniques 1145; 2024-12-03T06:13:40,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=20380, suffix heap size=147048,new segment heap size=46520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=445 2024-12-03T06:13:40,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=445, cellSize=20380, totalHeapSize=46520, min timestamp=1733206420669, max timestamp=1733206420712Num uniques 1145; 2024-12-03T06:13:40,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=20380, suffix heap size=147048,new segment heap size=46520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=445 2024-12-03T06:13:40,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733206420694, max timestamp=1733206420712Num uniques 1145; 2024-12-03T06:13:40,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-03T06:13:40,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741986_1162 (size=30955) 2024-12-03T06:13:40,735 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b914ffc0e3c544a3aed7e9287570f44e 2024-12-03T06:13:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/dc58fd6905cc4d5eb845b3aa99a1e924 is 50, key is test_row_0/B:col10/1733206419215/Put/seqid=0 2024-12-03T06:13:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206480756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741987_1163 (size=12001) 2024-12-03T06:13:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206480758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,765 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/dc58fd6905cc4d5eb845b3aa99a1e924 2024-12-03T06:13:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206480761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206480763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/1c292061b4654f71802981f011ee8c82 is 50, key is test_row_0/C:col10/1733206419215/Put/seqid=0 2024-12-03T06:13:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741988_1164 (size=12001) 2024-12-03T06:13:40,803 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/1c292061b4654f71802981f011ee8c82 2024-12-03T06:13:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b914ffc0e3c544a3aed7e9287570f44e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e 2024-12-03T06:13:40,826 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e, entries=150, sequenceid=104, filesize=30.2 K 2024-12-03T06:13:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/dc58fd6905cc4d5eb845b3aa99a1e924 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924 2024-12-03T06:13:40,832 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924, entries=150, sequenceid=104, filesize=11.7 K 2024-12-03T06:13:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/1c292061b4654f71802981f011ee8c82 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82 2024-12-03T06:13:40,840 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82, entries=150, sequenceid=104, filesize=11.7 K 2024-12-03T06:13:40,841 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=167.72 KB/171750 for 87b34dfe780d9655f141bfd2362b0642 in 629ms, sequenceid=104, compaction requested=false 2024-12-03T06:13:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T06:13:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T06:13:40,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T06:13:40,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-12-03T06:13:40,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7260 sec 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206420752, max timestamp=1733206420874Num uniques 1304; 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206420752, max timestamp=1733206420874Num uniques 1304; 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=59720, new segment data size=7050, suffix heap size=163584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1304, new segment cells count=154 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=59720, new segment data size=7050, suffix heap size=163584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1304, new segment cells count=154 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206420752, max timestamp=1733206420874Num uniques 1154; 2024-12-03T06:13:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-03T06:13:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:40,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=53.67 KB heapSize=146.39 KB 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:40,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203489db116c5704463a74b0dd1675ef3c3_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:40,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741989_1165 (size=14794) 2024-12-03T06:13:40,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420899, max timestamp=1733206420918Num uniques 859; 2024-12-03T06:13:40,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420899, max timestamp=1733206420918Num uniques 859; 2024-12-03T06:13:40,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206420899, max timestamp=1733206420918Num uniques 859; 2024-12-03T06:13:40,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:13:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206480956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206480957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206480958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:40,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206480963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206481069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206481070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206481070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206481080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T06:13:41,270 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:41,271 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T06:13:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206481270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,273 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 87b34dfe780d9655f141bfd2362b0642 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:41,274 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T06:13:41,275 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:41,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206481280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206481280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206481280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206481290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:41,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203489db116c5704463a74b0dd1675ef3c3_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203489db116c5704463a74b0dd1675ef3c3_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:41,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/94a7593fc5054eeea47828abcf1c30f8, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/94a7593fc5054eeea47828abcf1c30f8 is 175, key is test_row_0/A:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741990_1166 (size=39749) 2024-12-03T06:13:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T06:13:41,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T06:13:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T06:13:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T06:13:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206481599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206481600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206481600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206481600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:41,739 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/94a7593fc5054eeea47828abcf1c30f8 2024-12-03T06:13:41,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T06:13:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:41,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/fd166eab250049568a00387d13cefdf5 is 50, key is test_row_0/B:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741991_1167 (size=12151) 2024-12-03T06:13:41,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/fd166eab250049568a00387d13cefdf5 2024-12-03T06:13:41,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/5f8f1a03b8994ca1962c62afbfa8f5ba is 50, key is test_row_0/C:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741992_1168 (size=12151) 2024-12-03T06:13:41,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/5f8f1a03b8994ca1962c62afbfa8f5ba 2024-12-03T06:13:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/94a7593fc5054eeea47828abcf1c30f8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8 2024-12-03T06:13:41,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8, entries=200, sequenceid=219, filesize=38.8 K 2024-12-03T06:13:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/fd166eab250049568a00387d13cefdf5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5 2024-12-03T06:13:41,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T06:13:41,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/5f8f1a03b8994ca1962c62afbfa8f5ba as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba 2024-12-03T06:13:41,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T06:13:41,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~166.27 KB/170256, currentSize=127.47 KB/130530 for 87b34dfe780d9655f141bfd2362b0642 in 949ms, sequenceid=219, compaction requested=true 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:41,831 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:41,831 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:41,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:41,833 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,833 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,833 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=99.4 K 2024-12-03T06:13:41,833 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/5ba730daf2864576b30187ce4856af16, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=35.4 K 2024-12-03T06:13:41,833 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8] 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting df6afeff21984e05a16d61dd8ae5d2fe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:41,833 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba730daf2864576b30187ce4856af16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:41,834 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting dc58fd6905cc4d5eb845b3aa99a1e924, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733206419214 2024-12-03T06:13:41,834 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b914ffc0e3c544a3aed7e9287570f44e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733206419214 2024-12-03T06:13:41,834 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a7593fc5054eeea47828abcf1c30f8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420874 2024-12-03T06:13:41,834 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting fd166eab250049568a00387d13cefdf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420880 2024-12-03T06:13:41,855 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,868 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:41,869 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/2cd32d69fae54e5b9069b02de0e707ce is 50, key is test_row_0/B:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,871 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203aa9e0c42edd44c8b9ec236c95a2f84a5_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,873 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203aa9e0c42edd44c8b9ec236c95a2f84a5_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,874 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa9e0c42edd44c8b9ec236c95a2f84a5_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741993_1169 (size=4469) 2024-12-03T06:13:41,880 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#146 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:41,881 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/22b235e585224475907951ece2ece206 is 175, key is test_row_0/A:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741994_1170 (size=12391) 2024-12-03T06:13:41,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,897 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T06:13:41,904 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/2cd32d69fae54e5b9069b02de0e707ce as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/2cd32d69fae54e5b9069b02de0e707ce 2024-12-03T06:13:41,914 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 2cd32d69fae54e5b9069b02de0e707ce(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:41,914 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:41,914 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206421831; duration=0sec 2024-12-03T06:13:41,915 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:41,915 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:41,915 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:41,917 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:41,917 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:41,917 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:41,918 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9e9ef0f6b1f642e2977d40b2c46d2f2a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=35.4 K 2024-12-03T06:13:41,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741995_1171 (size=31345) 2024-12-03T06:13:41,922 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9ef0f6b1f642e2977d40b2c46d2f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733206418551 2024-12-03T06:13:41,923 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c292061b4654f71802981f011ee8c82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733206419214 2024-12-03T06:13:41,924 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8f1a03b8994ca1962c62afbfa8f5ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420880 2024-12-03T06:13:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f419d7bdebd4837b14ebdd2cae2fbba_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206420956/Put/seqid=0 2024-12-03T06:13:41,935 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/22b235e585224475907951ece2ece206 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206 2024-12-03T06:13:41,941 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 22b235e585224475907951ece2ece206(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:41,941 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:41,941 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206421831; duration=0sec 2024-12-03T06:13:41,941 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:41,941 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:41,948 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:41,948 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/1d3568356df04d51ab929d3bae2a8a77 is 50, key is test_row_0/C:col10/1733206420882/Put/seqid=0 2024-12-03T06:13:41,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741996_1172 (size=12304) 2024-12-03T06:13:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:41,959 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f419d7bdebd4837b14ebdd2cae2fbba_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f419d7bdebd4837b14ebdd2cae2fbba_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:41,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741997_1173 (size=12391) 2024-12-03T06:13:41,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e651ccf950ed4fd9be16e7ecf1aa8902, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:41,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e651ccf950ed4fd9be16e7ecf1aa8902 is 175, key is test_row_0/A:col10/1733206420956/Put/seqid=0 2024-12-03T06:13:41,969 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/1d3568356df04d51ab929d3bae2a8a77 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1d3568356df04d51ab929d3bae2a8a77 2024-12-03T06:13:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741998_1174 (size=31105) 2024-12-03T06:13:41,976 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e651ccf950ed4fd9be16e7ecf1aa8902 2024-12-03T06:13:41,977 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into 1d3568356df04d51ab929d3bae2a8a77(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:41,977 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:41,977 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=13, startTime=1733206421831; duration=0sec 2024-12-03T06:13:41,977 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:41,977 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:41,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/61da4ffc22e746048787bdda782d66d0 is 50, key is test_row_0/B:col10/1733206420956/Put/seqid=0 2024-12-03T06:13:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741999_1175 (size=12151) 2024-12-03T06:13:42,014 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/61da4ffc22e746048787bdda782d66d0 2024-12-03T06:13:42,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0558f23ab2f74c98b8b85f8bed05d191 is 50, key is test_row_0/C:col10/1733206420956/Put/seqid=0 2024-12-03T06:13:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742000_1176 (size=12151) 2024-12-03T06:13:42,045 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0558f23ab2f74c98b8b85f8bed05d191 2024-12-03T06:13:42,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/e651ccf950ed4fd9be16e7ecf1aa8902 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902 2024-12-03T06:13:42,061 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902, entries=150, sequenceid=254, filesize=30.4 K 2024-12-03T06:13:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/61da4ffc22e746048787bdda782d66d0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0 2024-12-03T06:13:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,067 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T06:13:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0558f23ab2f74c98b8b85f8bed05d191 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191 2024-12-03T06:13:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,075 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T06:13:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,082 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 87b34dfe780d9655f141bfd2362b0642 in 185ms, sequenceid=254, compaction requested=false 2024-12-03T06:13:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T06:13:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T06:13:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T06:13:42,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 809 msec 2024-12-03T06:13:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 817 msec 2024-12-03T06:13:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1afd638049f41d7a124a5ae1a379ffb_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742001_1177 (size=22618) 2024-12-03T06:13:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733206422154, max timestamp=1733206422200Num uniques 859; 2024-12-03T06:13:42,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T06:13:42,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733206422154, max timestamp=1733206422200Num uniques 859; 2024-12-03T06:13:42,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T06:13:42,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733206422154, max timestamp=1733206422200Num uniques 859; 2024-12-03T06:13:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T06:13:42,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206482204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206482206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206482208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206482209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206482320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206482320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206482320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206482320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T06:13:42,409 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:42,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-03T06:13:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:42,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:42,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:42,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:42,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206482529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206482540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206482540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206482540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:42,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1afd638049f41d7a124a5ae1a379ffb_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1afd638049f41d7a124a5ae1a379ffb_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:42,581 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/65962550a7f1457f8635f8986c92e874, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:42,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/65962550a7f1457f8635f8986c92e874 is 175, key is test_row_0/A:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742002_1178 (size=66023) 2024-12-03T06:13:42,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206482840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206482849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206482849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206482860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:42,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:42,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:42,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:42,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/65962550a7f1457f8635f8986c92e874 2024-12-03T06:13:43,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/511833d7ab1d47d3806d4e0d7ee3dc88 is 50, key is test_row_0/B:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:43,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742003_1179 (size=12301) 2024-12-03T06:13:43,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/511833d7ab1d47d3806d4e0d7ee3dc88 2024-12-03T06:13:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:43,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/4607eb052d5a49948f519285a6ae42e0 is 50, key is test_row_0/C:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:43,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742004_1180 (size=12301) 2024-12-03T06:13:43,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206483349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:43,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206483360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206483360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206483379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:43,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/4607eb052d5a49948f519285a6ae42e0 2024-12-03T06:13:43,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/65962550a7f1457f8635f8986c92e874 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874 2024-12-03T06:13:43,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874, entries=350, sequenceid=267, filesize=64.5 K 2024-12-03T06:13:43,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/511833d7ab1d47d3806d4e0d7ee3dc88 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88 2024-12-03T06:13:43,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T06:13:43,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/4607eb052d5a49948f519285a6ae42e0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0 2024-12-03T06:13:43,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T06:13:43,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 87b34dfe780d9655f141bfd2362b0642 in 1343ms, sequenceid=267, compaction requested=true 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:43,496 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:43,496 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:43,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:43,498 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:43,498 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:43,498 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T06:13:43,498 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/2cd32d69fae54e5b9069b02de0e707ce, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.0 K 2024-12-03T06:13:43,498 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,499 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:43,499 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,499 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-12-03T06:13:43,499 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=125.5 K 2024-12-03T06:13:43,499 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,499 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874] 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:43,500 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd32d69fae54e5b9069b02de0e707ce, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420880 2024-12-03T06:13:43,501 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 61da4ffc22e746048787bdda782d66d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206420948 2024-12-03T06:13:43,501 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 511833d7ab1d47d3806d4e0d7ee3dc88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206422140 2024-12-03T06:13:43,502 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b235e585224475907951ece2ece206, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420880 2024-12-03T06:13:43,504 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e651ccf950ed4fd9be16e7ecf1aa8902, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206420948 2024-12-03T06:13:43,507 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65962550a7f1457f8635f8986c92e874, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206422134 2024-12-03T06:13:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036fc84b393d6f4d8aa3fdb7bc34a6f4a2_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206422207/Put/seqid=0 2024-12-03T06:13:43,520 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:43,521 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:43,522 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/7f9175e0e59743fc94a1bb6d28d15718 is 50, key is test_row_0/B:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:43,526 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b87875e80fb242988eac129634aea08e_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:43,528 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b87875e80fb242988eac129634aea08e_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:43,528 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b87875e80fb242988eac129634aea08e_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:43,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742005_1181 (size=12454) 2024-12-03T06:13:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,537 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036fc84b393d6f4d8aa3fdb7bc34a6f4a2_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fc84b393d6f4d8aa3fdb7bc34a6f4a2_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:43,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742006_1182 (size=12643) 2024-12-03T06:13:43,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742007_1183 (size=4469) 2024-12-03T06:13:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/bf13b9a6546c458091f3ec5534f5a53d, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/bf13b9a6546c458091f3ec5534f5a53d is 175, key is test_row_0/A:col10/1733206422207/Put/seqid=0 2024-12-03T06:13:43,547 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#157 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:43,547 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/4fb12820f00b43c594bcefcde17eeabc is 175, key is test_row_0/A:col10/1733206422149/Put/seqid=0 2024-12-03T06:13:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:43,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742009_1185 (size=31597) 2024-12-03T06:13:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742008_1184 (size=31255) 2024-12-03T06:13:43,590 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/bf13b9a6546c458091f3ec5534f5a53d 2024-12-03T06:13:43,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ffceb975db614d01894cb8b862f58a5e is 50, key is test_row_0/B:col10/1733206422207/Put/seqid=0 2024-12-03T06:13:43,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742010_1186 (size=12301) 2024-12-03T06:13:43,608 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ffceb975db614d01894cb8b862f58a5e 2024-12-03T06:13:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/526c3578684e461b94132cbd08981098 is 50, key is test_row_0/C:col10/1733206422207/Put/seqid=0 2024-12-03T06:13:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742011_1187 (size=12301) 2024-12-03T06:13:43,629 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/526c3578684e461b94132cbd08981098 2024-12-03T06:13:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/bf13b9a6546c458091f3ec5534f5a53d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d 2024-12-03T06:13:43,641 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d, entries=150, sequenceid=293, filesize=30.5 K 2024-12-03T06:13:43,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ffceb975db614d01894cb8b862f58a5e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e 2024-12-03T06:13:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,647 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e, entries=150, sequenceid=293, filesize=12.0 K 2024-12-03T06:13:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/526c3578684e461b94132cbd08981098 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098 2024-12-03T06:13:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,654 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098, entries=150, sequenceid=293, filesize=12.0 K 2024-12-03T06:13:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,655 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=0 B/0 for 87b34dfe780d9655f141bfd2362b0642 in 156ms, sequenceid=293, compaction requested=true 2024-12-03T06:13:43,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:43,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-03T06:13:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-03T06:13:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T06:13:43,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2350 sec 2024-12-03T06:13:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2510 sec 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,949 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/7f9175e0e59743fc94a1bb6d28d15718 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7f9175e0e59743fc94a1bb6d28d15718 2024-12-03T06:13:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,956 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 7f9175e0e59743fc94a1bb6d28d15718(size=12.3 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:43,956 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:43,956 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206423496; duration=0sec 2024-12-03T06:13:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,957 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,957 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:43,957 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,959 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,959 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,959 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:43,959 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1d3568356df04d51ab929d3bae2a8a77, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=48.0 K 2024-12-03T06:13:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,960 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3568356df04d51ab929d3bae2a8a77, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733206420880 2024-12-03T06:13:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,960 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0558f23ab2f74c98b8b85f8bed05d191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206420948 2024-12-03T06:13:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,961 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4607eb052d5a49948f519285a6ae42e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206422140 2024-12-03T06:13:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,961 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 526c3578684e461b94132cbd08981098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733206422203 2024-12-03T06:13:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,976 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/4fb12820f00b43c594bcefcde17eeabc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc 2024-12-03T06:13:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,980 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,981 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/098a86da6df24de096925901a6c3c15f is 50, key is test_row_0/C:col10/1733206422207/Put/seqid=0 2024-12-03T06:13:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,984 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 4fb12820f00b43c594bcefcde17eeabc(size=30.9 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,984 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:43,984 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206423496; duration=0sec 2024-12-03T06:13:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,984 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:43,984 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742012_1188 (size=12677) 2024-12-03T06:13:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,403 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/098a86da6df24de096925901a6c3c15f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/098a86da6df24de096925901a6c3c15f 2024-12-03T06:13:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,410 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into 098a86da6df24de096925901a6c3c15f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:44,410 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:44,410 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=12, startTime=1733206423496; duration=0sec 2024-12-03T06:13:44,410 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:44,410 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:44,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:13:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120344e648bf6ef444948b27f73681828c90_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742013_1189 (size=22618) 2024-12-03T06:13:44,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,471 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120344e648bf6ef444948b27f73681828c90_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120344e648bf6ef444948b27f73681828c90_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,472 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b266e72a85684a188174f5ddf60a347e, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:44,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b266e72a85684a188174f5ddf60a347e is 175, key is test_row_0/A:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742014_1190 (size=66023) 2024-12-03T06:13:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:44,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b266e72a85684a188174f5ddf60a347e 2024-12-03T06:13:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/f4559f4462984530876ae41dcb708f52 is 50, key is test_row_0/B:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:44,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742015_1191 (size=12301) 2024-12-03T06:13:44,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/f4559f4462984530876ae41dcb708f52 2024-12-03T06:13:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206424488, max timestamp=1733206424503Num uniques 859; 2024-12-03T06:13:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733206424488, max timestamp=1733206424503Num uniques 859; 2024-12-03T06:13:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T06:13:44,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206424488, max timestamp=1733206424503Num uniques 859; 2024-12-03T06:13:44,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:13:44,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/459016469658465994dd4273660cc8b7 is 50, key is test_row_0/C:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:44,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742016_1192 (size=12301) 2024-12-03T06:13:44,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206484542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206484542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206484543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206484550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T06:13:44,560 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:44,561 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-03T06:13:44,564 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:44,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T06:13:44,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206484659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206484659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206484660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206484660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T06:13:44,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T06:13:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T06:13:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206484870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206484870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206484870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206484871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T06:13:44,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/459016469658465994dd4273660cc8b7 2024-12-03T06:13:44,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/b266e72a85684a188174f5ddf60a347e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e 2024-12-03T06:13:44,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e, entries=350, sequenceid=306, filesize=64.5 K 2024-12-03T06:13:44,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/f4559f4462984530876ae41dcb708f52 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52 2024-12-03T06:13:44,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T06:13:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/459016469658465994dd4273660cc8b7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7 2024-12-03T06:13:44,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T06:13:44,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 87b34dfe780d9655f141bfd2362b0642 in 523ms, sequenceid=306, compaction requested=true 2024-12-03T06:13:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:44,955 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:44,955 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:44,956 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:44,956 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128875 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:44,957 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:44,957 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:44,957 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,957 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,957 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=125.9 K 2024-12-03T06:13:44,957 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7f9175e0e59743fc94a1bb6d28d15718, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.4 K 2024-12-03T06:13:44,957 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:44,957 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e] 2024-12-03T06:13:44,957 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9175e0e59743fc94a1bb6d28d15718, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206422140 2024-12-03T06:13:44,957 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fb12820f00b43c594bcefcde17eeabc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206422140 2024-12-03T06:13:44,958 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ffceb975db614d01894cb8b862f58a5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733206422203 2024-12-03T06:13:44,958 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf13b9a6546c458091f3ec5534f5a53d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733206422203 2024-12-03T06:13:44,958 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f4559f4462984530876ae41dcb708f52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733206424424 2024-12-03T06:13:44,958 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b266e72a85684a188174f5ddf60a347e, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733206424391 2024-12-03T06:13:44,970 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:44,971 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:44,972 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/dafd09765d7a410eafba39da2d302826 is 50, key is test_row_0/B:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:44,978 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c0d7ddbff35245b68824203f6b645b7e_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:44,980 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c0d7ddbff35245b68824203f6b645b7e_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:44,981 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c0d7ddbff35245b68824203f6b645b7e_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:44,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742017_1193 (size=12745) 2024-12-03T06:13:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742018_1194 (size=4469) 2024-12-03T06:13:45,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T06:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:45,026 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-03T06:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:13:45,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037be7498fcbcf47ce8e35baea5000a013_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206424541/Put/seqid=0 2024-12-03T06:13:45,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742019_1195 (size=12454) 2024-12-03T06:13:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:45,053 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037be7498fcbcf47ce8e35baea5000a013_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037be7498fcbcf47ce8e35baea5000a013_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:45,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/ed44c19be2244d1eb6dee2febb179e11, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:45,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/ed44c19be2244d1eb6dee2febb179e11 is 175, key is test_row_0/A:col10/1733206424541/Put/seqid=0 2024-12-03T06:13:45,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742020_1196 (size=31255) 2024-12-03T06:13:45,060 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/ed44c19be2244d1eb6dee2febb179e11 2024-12-03T06:13:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/e6cee1c3934f430bbba85e2bd9a4354f is 50, key is test_row_0/B:col10/1733206424541/Put/seqid=0 2024-12-03T06:13:45,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742021_1197 (size=12301) 2024-12-03T06:13:45,078 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/e6cee1c3934f430bbba85e2bd9a4354f 2024-12-03T06:13:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d1c437bd19f7482ebf38d59008d86abf is 50, key is test_row_0/C:col10/1733206424541/Put/seqid=0 2024-12-03T06:13:45,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742022_1198 (size=12301) 2024-12-03T06:13:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:45,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T06:13:45,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206485199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206485199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206485199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206485200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206485310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206485310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206485310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206485310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206485319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,320 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 87b34dfe780d9655f141bfd2362b0642 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,402 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/dafd09765d7a410eafba39da2d302826 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dafd09765d7a410eafba39da2d302826 2024-12-03T06:13:45,405 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:45,406 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/7f7e10a4319f4f6496ddebfbb647c93d is 175, key is test_row_0/A:col10/1733206424424/Put/seqid=0 2024-12-03T06:13:45,409 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into dafd09765d7a410eafba39da2d302826(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:45,409 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:45,409 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206424955; duration=0sec 2024-12-03T06:13:45,409 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:45,409 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:45,410 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T06:13:45,412 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T06:13:45,412 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T06:13:45,412 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. because compaction request was cancelled 2024-12-03T06:13:45,412 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:45,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742023_1199 (size=31699) 2024-12-03T06:13:45,421 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/7f7e10a4319f4f6496ddebfbb647c93d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d 2024-12-03T06:13:45,429 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 7f7e10a4319f4f6496ddebfbb647c93d(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:45,430 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:45,430 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206424955; duration=0sec 2024-12-03T06:13:45,430 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:45,430 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:45,504 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d1c437bd19f7482ebf38d59008d86abf 2024-12-03T06:13:45,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/ed44c19be2244d1eb6dee2febb179e11 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11 2024-12-03T06:13:45,516 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11, entries=150, sequenceid=343, filesize=30.5 K 2024-12-03T06:13:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/e6cee1c3934f430bbba85e2bd9a4354f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f 2024-12-03T06:13:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206485520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206485520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206485520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,525 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T06:13:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d1c437bd19f7482ebf38d59008d86abf as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf 2024-12-03T06:13:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206485530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,534 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T06:13:45,535 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 87b34dfe780d9655f141bfd2362b0642 in 509ms, sequenceid=343, compaction requested=true 2024-12-03T06:13:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-03T06:13:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-03T06:13:45,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T06:13:45,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 972 msec 2024-12-03T06:13:45,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 979 msec 2024-12-03T06:13:45,565 INFO [master/122cddc95ec8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T06:13:45,565 INFO [master/122cddc95ec8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T06:13:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T06:13:45,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:45,700 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-03T06:13:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:45,702 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:45,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:45,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:45,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:45,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203038a6bb6a0314b699aea4df2735c5663_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:45,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:45,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:45,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:45,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742024_1200 (size=12454) 2024-12-03T06:13:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:45,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206485894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206485894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206485895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206485900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206486010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206486010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206486010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206486010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:46,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206486219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206486219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206486219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206486230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:46,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203038a6bb6a0314b699aea4df2735c5663_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203038a6bb6a0314b699aea4df2735c5663_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:46,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/3bba573b049745e98e47f8f1c4269f61, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:46,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/3bba573b049745e98e47f8f1c4269f61 is 175, key is test_row_0/A:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:46,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742025_1201 (size=31255) 2024-12-03T06:13:46,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:46,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206486530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206486530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206486530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206486539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:46,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/3bba573b049745e98e47f8f1c4269f61 2024-12-03T06:13:46,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/6b003e565a1e44ae8b3fc907e633d70d is 50, key is test_row_0/B:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:46,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742026_1202 (size=12301) 2024-12-03T06:13:46,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:46,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:46,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206487040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206487040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206487040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206487049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:47,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/6b003e565a1e44ae8b3fc907e633d70d 2024-12-03T06:13:47,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/720f3c2e5ccb45dda44d4de024846773 is 50, key is test_row_0/C:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742027_1203 (size=12301) 2024-12-03T06:13:47,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/720f3c2e5ccb45dda44d4de024846773 2024-12-03T06:13:47,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/3bba573b049745e98e47f8f1c4269f61 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61 2024-12-03T06:13:47,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61, entries=150, sequenceid=358, filesize=30.5 K 2024-12-03T06:13:47,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/6b003e565a1e44ae8b3fc907e633d70d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d 2024-12-03T06:13:47,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d, entries=150, sequenceid=358, filesize=12.0 K 2024-12-03T06:13:47,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/720f3c2e5ccb45dda44d4de024846773 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773 2024-12-03T06:13:47,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:47,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773, entries=150, sequenceid=358, filesize=12.0 K 2024-12-03T06:13:47,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 87b34dfe780d9655f141bfd2362b0642 in 1711ms, sequenceid=358, compaction requested=true 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:47,543 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:47,543 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:47,545 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:47,545 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:47,545 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,545 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dafd09765d7a410eafba39da2d302826, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.5 K 2024-12-03T06:13:47,545 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:47,545 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:47,545 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,545 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=92.0 K 2024-12-03T06:13:47,546 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,546 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61] 2024-12-03T06:13:47,548 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting dafd09765d7a410eafba39da2d302826, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733206424424 2024-12-03T06:13:47,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7e10a4319f4f6496ddebfbb647c93d, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733206424424 2024-12-03T06:13:47,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e6cee1c3934f430bbba85e2bd9a4354f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733206424539 2024-12-03T06:13:47,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed44c19be2244d1eb6dee2febb179e11, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733206424539 2024-12-03T06:13:47,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b003e565a1e44ae8b3fc907e633d70d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:47,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bba573b049745e98e47f8f1c4269f61, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:47,558 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:47,567 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203956e0d1eb410433191e948e4ee1cfbde_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:47,568 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:47,569 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203956e0d1eb410433191e948e4ee1cfbde_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:47,569 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/71ab9db7eb6449f9af4fd6daf97005a0 is 50, key is test_row_0/B:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:47,569 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203956e0d1eb410433191e948e4ee1cfbde_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:47,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742028_1204 (size=12847) 2024-12-03T06:13:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742029_1205 (size=4469) 2024-12-03T06:13:47,603 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#172 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:47,604 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/727e40ccb13b474d9e031b6ea0b489c6 is 175, key is test_row_0/A:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:47,609 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/71ab9db7eb6449f9af4fd6daf97005a0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/71ab9db7eb6449f9af4fd6daf97005a0 2024-12-03T06:13:47,619 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 71ab9db7eb6449f9af4fd6daf97005a0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:47,619 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:47,620 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206427543; duration=0sec 2024-12-03T06:13:47,620 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:47,620 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:47,620 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:13:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742030_1206 (size=31801) 2024-12-03T06:13:47,621 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:13:47,622 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:47,622 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,622 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/098a86da6df24de096925901a6c3c15f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=48.4 K 2024-12-03T06:13:47,622 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 098a86da6df24de096925901a6c3c15f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733206422203 2024-12-03T06:13:47,623 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 459016469658465994dd4273660cc8b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733206424424 2024-12-03T06:13:47,623 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c437bd19f7482ebf38d59008d86abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733206424539 2024-12-03T06:13:47,623 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 720f3c2e5ccb45dda44d4de024846773, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:47,631 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:47,632 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a5f31e29ba304f5aa47551fc8dc88a60 is 50, key is test_row_0/C:col10/1733206425831/Put/seqid=0 2024-12-03T06:13:47,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742031_1207 (size=12813) 2024-12-03T06:13:47,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T06:13:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:47,697 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:47,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120359805097a3f64e7b9105f6ba9e3cc23e_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206425898/Put/seqid=0 2024-12-03T06:13:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742032_1208 (size=12454) 2024-12-03T06:13:47,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:47,718 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120359805097a3f64e7b9105f6ba9e3cc23e_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120359805097a3f64e7b9105f6ba9e3cc23e_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:47,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/fb7e29aacfde4dbe8e1a56dafd36e95e, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:47,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/fb7e29aacfde4dbe8e1a56dafd36e95e is 175, key is test_row_0/A:col10/1733206425898/Put/seqid=0 2024-12-03T06:13:47,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742033_1209 (size=31255) 2024-12-03T06:13:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:48,027 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/727e40ccb13b474d9e031b6ea0b489c6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6 2024-12-03T06:13:48,039 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 727e40ccb13b474d9e031b6ea0b489c6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:48,039 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:48,039 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206427543; duration=0sec 2024-12-03T06:13:48,039 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:48,039 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:48,064 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a5f31e29ba304f5aa47551fc8dc88a60 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a5f31e29ba304f5aa47551fc8dc88a60 2024-12-03T06:13:48,075 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into a5f31e29ba304f5aa47551fc8dc88a60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:48,075 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:48,075 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=12, startTime=1733206427543; duration=0sec 2024-12-03T06:13:48,075 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:48,075 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206488068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206488068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206488068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206488069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,132 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/fb7e29aacfde4dbe8e1a56dafd36e95e 2024-12-03T06:13:48,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/537f704c3b71494db3ca94e3b6e66825 is 50, key is test_row_0/B:col10/1733206425898/Put/seqid=0 2024-12-03T06:13:48,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742034_1210 (size=12301) 2024-12-03T06:13:48,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206488179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206488179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206488190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206488191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206488389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206488389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206488399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206488400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,552 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/537f704c3b71494db3ca94e3b6e66825 2024-12-03T06:13:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc is 50, key is test_row_0/C:col10/1733206425898/Put/seqid=0 2024-12-03T06:13:48,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742035_1211 (size=12301) 2024-12-03T06:13:48,585 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc 2024-12-03T06:13:48,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/fb7e29aacfde4dbe8e1a56dafd36e95e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e 2024-12-03T06:13:48,596 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e, entries=150, sequenceid=378, filesize=30.5 K 2024-12-03T06:13:48,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/537f704c3b71494db3ca94e3b6e66825 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825 2024-12-03T06:13:48,603 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825, entries=150, sequenceid=378, filesize=12.0 K 2024-12-03T06:13:48,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc 2024-12-03T06:13:48,613 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc, entries=150, sequenceid=378, filesize=12.0 K 2024-12-03T06:13:48,616 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87b34dfe780d9655f141bfd2362b0642 in 919ms, sequenceid=378, compaction requested=false 2024-12-03T06:13:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-03T06:13:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-03T06:13:48,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T06:13:48,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9160 sec 2024-12-03T06:13:48,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.9230 sec 2024-12-03T06:13:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:48,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:48,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120325eaaca9686f46f7ad5b453ef29190db_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:48,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742036_1212 (size=17534) 2024-12-03T06:13:48,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206488732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206488735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206488738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206488738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206488839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206488849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206488850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206488850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206489049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206489060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206489060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206489060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:49,136 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120325eaaca9686f46f7ad5b453ef29190db_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120325eaaca9686f46f7ad5b453ef29190db_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:49,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1acbb0fce37467780c0b8f63b7e0628, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:49,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1acbb0fce37467780c0b8f63b7e0628 is 175, key is test_row_0/A:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742037_1213 (size=48639) 2024-12-03T06:13:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206489359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206489370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206489370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206489370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1acbb0fce37467780c0b8f63b7e0628 2024-12-03T06:13:49,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee8d6c9b3daa4014932b89bbddd3f888 is 50, key is test_row_0/B:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:49,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742038_1214 (size=12301) 2024-12-03T06:13:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T06:13:49,870 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:49,870 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-03T06:13:49,872 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:49,872 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:49,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206489869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206489890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206489890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206489890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:49,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee8d6c9b3daa4014932b89bbddd3f888 2024-12-03T06:13:49,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d7b2d760433140aab21b97764fd6652a is 50, key is test_row_0/C:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:49,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742039_1215 (size=12301) 2024-12-03T06:13:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:50,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T06:13:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T06:13:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:50,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T06:13:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:50,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d7b2d760433140aab21b97764fd6652a 2024-12-03T06:13:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1acbb0fce37467780c0b8f63b7e0628 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628 2024-12-03T06:13:50,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628, entries=250, sequenceid=395, filesize=47.5 K 2024-12-03T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee8d6c9b3daa4014932b89bbddd3f888 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888 2024-12-03T06:13:50,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T06:13:50,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/d7b2d760433140aab21b97764fd6652a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a 2024-12-03T06:13:50,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T06:13:50,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 87b34dfe780d9655f141bfd2362b0642 in 1695ms, sequenceid=395, compaction requested=true 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:50,402 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:50,402 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:50,403 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:50,403 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:50,403 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:50,403 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:50,403 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,403 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,404 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=109.1 K 2024-12-03T06:13:50,404 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/71ab9db7eb6449f9af4fd6daf97005a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.6 K 2024-12-03T06:13:50,404 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,404 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628] 2024-12-03T06:13:50,404 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ab9db7eb6449f9af4fd6daf97005a0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:50,404 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727e40ccb13b474d9e031b6ea0b489c6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:50,404 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 537f704c3b71494db3ca94e3b6e66825, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733206425859 2024-12-03T06:13:50,404 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7e29aacfde4dbe8e1a56dafd36e95e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733206425859 2024-12-03T06:13:50,405 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1acbb0fce37467780c0b8f63b7e0628, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428065 2024-12-03T06:13:50,405 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8d6c9b3daa4014932b89bbddd3f888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428066 2024-12-03T06:13:50,414 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:50,415 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:50,415 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/230e632f79a94ea183913c93a0f2749a is 50, key is test_row_0/B:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:50,418 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203976523a153b74339948dfabb4b4e35b1_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:50,420 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203976523a153b74339948dfabb4b4e35b1_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:50,420 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203976523a153b74339948dfabb4b4e35b1_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:50,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742040_1216 (size=12949) 2024-12-03T06:13:50,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742041_1217 (size=4469) 2024-12-03T06:13:50,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T06:13:50,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,485 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:50,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037ddfcedabb1b4aba8106059561bad3cc_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206428728/Put/seqid=0 2024-12-03T06:13:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:50,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742042_1218 (size=12454) 2024-12-03T06:13:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:50,510 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037ddfcedabb1b4aba8106059561bad3cc_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ddfcedabb1b4aba8106059561bad3cc_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/0198851428f3401d9cea5e567701466d, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/0198851428f3401d9cea5e567701466d is 175, key is test_row_0/A:col10/1733206428728/Put/seqid=0 2024-12-03T06:13:50,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742043_1219 (size=31255) 2024-12-03T06:13:50,832 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/230e632f79a94ea183913c93a0f2749a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/230e632f79a94ea183913c93a0f2749a 2024-12-03T06:13:50,838 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#182 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:50,838 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/4efce3925353443b8306bd1371834c0a is 175, key is test_row_0/A:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:50,840 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 230e632f79a94ea183913c93a0f2749a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:50,840 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:50,840 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206430402; duration=0sec 2024-12-03T06:13:50,840 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:50,840 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:50,840 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:50,848 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:50,848 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:50,848 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:50,848 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a5f31e29ba304f5aa47551fc8dc88a60, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.5 K 2024-12-03T06:13:50,849 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f31e29ba304f5aa47551fc8dc88a60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733206425199 2024-12-03T06:13:50,850 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a62ab5cc8c2d4cfbb55d8af73b8c21bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733206425859 2024-12-03T06:13:50,850 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b2d760433140aab21b97764fd6652a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428066 2024-12-03T06:13:50,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742044_1220 (size=31903) 2024-12-03T06:13:50,867 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:50,867 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9 is 50, key is test_row_0/C:col10/1733206428066/Put/seqid=0 2024-12-03T06:13:50,868 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/4efce3925353443b8306bd1371834c0a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a 2024-12-03T06:13:50,874 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 4efce3925353443b8306bd1371834c0a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:50,874 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:50,874 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206430402; duration=0sec 2024-12-03T06:13:50,874 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:50,874 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:50,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742045_1221 (size=12915) 2024-12-03T06:13:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:50,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:50,924 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/0198851428f3401d9cea5e567701466d 2024-12-03T06:13:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206490920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206490924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206490925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206490925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/975ad8527f57433896329d9418e9623c is 50, key is test_row_0/B:col10/1733206428728/Put/seqid=0 2024-12-03T06:13:50,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742046_1222 (size=12301) 2024-12-03T06:13:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206491030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206491040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206491040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206491040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206491239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206491250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206491250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206491250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,284 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9 2024-12-03T06:13:51,292 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into 6cffaf301aeb4ed8b7c11f21ac7cb8e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:51,292 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:51,292 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=13, startTime=1733206430402; duration=0sec 2024-12-03T06:13:51,292 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:51,292 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:51,338 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/975ad8527f57433896329d9418e9623c 2024-12-03T06:13:51,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/9808c975676540cea88324852c4035f3 is 50, key is test_row_0/C:col10/1733206428728/Put/seqid=0 2024-12-03T06:13:51,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742047_1223 (size=12301) 2024-12-03T06:13:51,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206491549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206491560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206491560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206491561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:51,752 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/9808c975676540cea88324852c4035f3 2024-12-03T06:13:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/0198851428f3401d9cea5e567701466d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d 2024-12-03T06:13:51,763 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d, entries=150, sequenceid=413, filesize=30.5 K 2024-12-03T06:13:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/975ad8527f57433896329d9418e9623c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c 2024-12-03T06:13:51,769 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T06:13:51,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/9808c975676540cea88324852c4035f3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3 2024-12-03T06:13:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T06:13:51,778 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 87b34dfe780d9655f141bfd2362b0642 in 1293ms, sequenceid=413, compaction requested=false 2024-12-03T06:13:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-03T06:13:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-03T06:13:51,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T06:13:51,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-12-03T06:13:51,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9190 sec 2024-12-03T06:13:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T06:13:52,020 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:52,020 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-03T06:13:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:52,022 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:52,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:52,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:52,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:52,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c41c8ca95840440b8a3d2ae92cc6cb43_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:52,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742048_1224 (size=14994) 2024-12-03T06:13:52,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206492085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206492086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206492088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206492089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,093 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c41c8ca95840440b8a3d2ae92cc6cb43_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c41c8ca95840440b8a3d2ae92cc6cb43_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:52,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c8c51ef93b3744eeab433ae7eb15c075, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:52,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c8c51ef93b3744eeab433ae7eb15c075 is 175, key is test_row_0/A:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:52,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742049_1225 (size=39949) 2024-12-03T06:13:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:52,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206492200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206492199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206492200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206492200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:52,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206492409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206492410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206492410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206492410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c8c51ef93b3744eeab433ae7eb15c075 2024-12-03T06:13:52,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/64fcb60e96bd434b9ce2c7d919e116e2 is 50, key is test_row_0/B:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:52,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742050_1226 (size=12301) 2024-12-03T06:13:52,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206492720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206492720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206492720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206492720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:52,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/64fcb60e96bd434b9ce2c7d919e116e2 2024-12-03T06:13:52,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/be2bc4938074478a9771691f3ec9beeb is 50, key is test_row_0/C:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:52,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742051_1227 (size=12301) 2024-12-03T06:13:52,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:52,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206493229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206493229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:53,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206493229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206493239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:53,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:13:53,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/be2bc4938074478a9771691f3ec9beeb 2024-12-03T06:13:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/c8c51ef93b3744eeab433ae7eb15c075 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075 2024-12-03T06:13:53,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075, entries=200, sequenceid=431, filesize=39.0 K 2024-12-03T06:13:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/64fcb60e96bd434b9ce2c7d919e116e2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2 2024-12-03T06:13:53,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T06:13:53,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/be2bc4938074478a9771691f3ec9beeb as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb 2024-12-03T06:13:53,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T06:13:53,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87b34dfe780d9655f141bfd2362b0642 in 1292ms, sequenceid=431, compaction requested=true 2024-12-03T06:13:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:53,355 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:53,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:53,356 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:53,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:53,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:53,357 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,357 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,357 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/230e632f79a94ea183913c93a0f2749a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.7 K 2024-12-03T06:13:53,357 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=100.7 K 2024-12-03T06:13:53,357 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075] 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 230e632f79a94ea183913c93a0f2749a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428066 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4efce3925353443b8306bd1371834c0a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428066 2024-12-03T06:13:53,357 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 975ad8527f57433896329d9418e9623c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733206428728 2024-12-03T06:13:53,358 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0198851428f3401d9cea5e567701466d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733206428728 2024-12-03T06:13:53,358 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 64fcb60e96bd434b9ce2c7d919e116e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:53,358 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c51ef93b3744eeab433ae7eb15c075, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:53,376 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:53,377 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:53,377 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/abac5e7778ab416fbec28f97222e1fcd is 50, key is test_row_0/B:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:53,381 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033ded2e9591b345ef97b1db2f91e60919_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:53,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742052_1228 (size=13051) 2024-12-03T06:13:53,383 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033ded2e9591b345ef97b1db2f91e60919_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:53,383 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033ded2e9591b345ef97b1db2f91e60919_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742053_1229 (size=4469) 2024-12-03T06:13:53,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,406 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:53,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036a9593dbcc174b22968cfaf3ef0ad11c_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206432082/Put/seqid=0 2024-12-03T06:13:53,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742054_1230 (size=12454) 2024-12-03T06:13:53,792 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/abac5e7778ab416fbec28f97222e1fcd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/abac5e7778ab416fbec28f97222e1fcd 2024-12-03T06:13:53,793 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#191 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:53,794 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/936a0e55be6945e381750136fb7f3f41 is 175, key is test_row_0/A:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:53,799 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into abac5e7778ab416fbec28f97222e1fcd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:53,799 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:53,799 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206433355; duration=0sec 2024-12-03T06:13:53,799 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:53,799 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:53,799 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:53,801 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:53,801 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:53,801 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:53,801 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.6 K 2024-12-03T06:13:53,803 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cffaf301aeb4ed8b7c11f21ac7cb8e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733206428066 2024-12-03T06:13:53,804 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9808c975676540cea88324852c4035f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733206428728 2024-12-03T06:13:53,804 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting be2bc4938074478a9771691f3ec9beeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:53,829 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036a9593dbcc174b22968cfaf3ef0ad11c_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a9593dbcc174b22968cfaf3ef0ad11c_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:53,830 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/04bc445c29354c3da4b7cc72d2857bb7, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:53,831 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/6379482aaa15430bb0feb8812d961ba8 is 50, key is test_row_0/C:col10/1733206430916/Put/seqid=0 2024-12-03T06:13:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/04bc445c29354c3da4b7cc72d2857bb7 is 175, key is test_row_0/A:col10/1733206432082/Put/seqid=0 2024-12-03T06:13:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742055_1231 (size=32005) 2024-12-03T06:13:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742056_1232 (size=13017) 2024-12-03T06:13:53,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742057_1233 (size=31255) 2024-12-03T06:13:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:54,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:54,245 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/936a0e55be6945e381750136fb7f3f41 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41 2024-12-03T06:13:54,253 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 936a0e55be6945e381750136fb7f3f41(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:54,253 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:54,254 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206433355; duration=0sec 2024-12-03T06:13:54,254 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:54,254 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:54,256 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/6379482aaa15430bb0feb8812d961ba8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6379482aaa15430bb0feb8812d961ba8 2024-12-03T06:13:54,263 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/04bc445c29354c3da4b7cc72d2857bb7 2024-12-03T06:13:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/7ca74c478f1244baa9de5903686857f6 is 50, key is test_row_0/B:col10/1733206432082/Put/seqid=0 2024-12-03T06:13:54,272 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into 6379482aaa15430bb0feb8812d961ba8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:54,272 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:54,273 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=13, startTime=1733206433356; duration=0sec 2024-12-03T06:13:54,273 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:54,273 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742058_1234 (size=12301) 2024-12-03T06:13:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206494282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206494284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206494286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206494285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206494390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206494399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206494399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206494399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206494600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206494609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206494609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206494609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,677 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/7ca74c478f1244baa9de5903686857f6 2024-12-03T06:13:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a221271be54e4a9082c8fb2c2b3153d6 is 50, key is test_row_0/C:col10/1733206432082/Put/seqid=0 2024-12-03T06:13:54,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742059_1235 (size=12301) 2024-12-03T06:13:54,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206494909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206494920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206494920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206494920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,089 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a221271be54e4a9082c8fb2c2b3153d6 2024-12-03T06:13:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/04bc445c29354c3da4b7cc72d2857bb7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7 2024-12-03T06:13:55,105 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7, entries=150, sequenceid=448, filesize=30.5 K 2024-12-03T06:13:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/7ca74c478f1244baa9de5903686857f6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6 2024-12-03T06:13:55,117 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6, entries=150, sequenceid=448, filesize=12.0 K 2024-12-03T06:13:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/a221271be54e4a9082c8fb2c2b3153d6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6 2024-12-03T06:13:55,124 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6, entries=150, sequenceid=448, filesize=12.0 K 2024-12-03T06:13:55,125 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 87b34dfe780d9655f141bfd2362b0642 in 1719ms, sequenceid=448, compaction requested=false 2024-12-03T06:13:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T06:13:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T06:13:55,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T06:13:55,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1040 sec 2024-12-03T06:13:55,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.1090 sec 2024-12-03T06:13:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:55,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:55,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033976ed3c3b334fae80ff94181c2104e1_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:55,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742060_1236 (size=12454) 2024-12-03T06:13:55,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206495428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206495429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206495429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206495430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206495433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206495541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206495541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206495749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206495749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:55,809 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:55,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033976ed3c3b334fae80ff94181c2104e1_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033976ed3c3b334fae80ff94181c2104e1_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:55,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/99404a1e77144b3fa8e59b05c6e126b8, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:55,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/99404a1e77144b3fa8e59b05c6e126b8 is 175, key is test_row_0/A:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:55,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742061_1237 (size=31255) 2024-12-03T06:13:56,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206496059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206496059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T06:13:56,180 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:56,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:13:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-03T06:13:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:56,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:13:56,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:13:56,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:13:56,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/99404a1e77144b3fa8e59b05c6e126b8 2024-12-03T06:13:56,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee72be29b5664dfa94d6bf3d197668d4 is 50, key is test_row_0/B:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742062_1238 (size=12301) 2024-12-03T06:13:56,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee72be29b5664dfa94d6bf3d197668d4 2024-12-03T06:13:56,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/e225ebb9d7274d7cbf5290a4626ac8f1 is 50, key is test_row_0/C:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:56,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742063_1239 (size=12301) 2024-12-03T06:13:56,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/e225ebb9d7274d7cbf5290a4626ac8f1 2024-12-03T06:13:56,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/99404a1e77144b3fa8e59b05c6e126b8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8 2024-12-03T06:13:56,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8, entries=150, sequenceid=468, filesize=30.5 K 2024-12-03T06:13:56,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/ee72be29b5664dfa94d6bf3d197668d4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4 2024-12-03T06:13:56,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T06:13:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/e225ebb9d7274d7cbf5290a4626ac8f1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1 2024-12-03T06:13:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:56,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T06:13:56,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 87b34dfe780d9655f141bfd2362b0642 in 899ms, sequenceid=468, compaction requested=true 2024-12-03T06:13:56,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:56,294 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:13:56,294 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:56,295 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:56,295 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/A is initiating minor compaction (all files) 2024-12-03T06:13:56,295 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/A in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:56,296 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=92.3 K 2024-12-03T06:13:56,296 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:56,296 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8] 2024-12-03T06:13:56,296 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:56,296 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/B is initiating minor compaction (all files) 2024-12-03T06:13:56,296 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/B in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:56,296 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/abac5e7778ab416fbec28f97222e1fcd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.8 K 2024-12-03T06:13:56,296 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936a0e55be6945e381750136fb7f3f41, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:56,297 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting abac5e7778ab416fbec28f97222e1fcd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:56,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04bc445c29354c3da4b7cc72d2857bb7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733206432082 2024-12-03T06:13:56,297 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca74c478f1244baa9de5903686857f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733206432082 2024-12-03T06:13:56,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99404a1e77144b3fa8e59b05c6e126b8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733206434240 2024-12-03T06:13:56,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ee72be29b5664dfa94d6bf3d197668d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733206434240 2024-12-03T06:13:56,314 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:56,323 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#B#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:56,324 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/965d335019e84518aa733b4e1ff9aa6e is 50, key is test_row_0/B:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:56,326 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203bb6b95763e3d40439b9f10cfd096ec17_87b34dfe780d9655f141bfd2362b0642 store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:56,328 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203bb6b95763e3d40439b9f10cfd096ec17_87b34dfe780d9655f141bfd2362b0642, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:56,328 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bb6b95763e3d40439b9f10cfd096ec17_87b34dfe780d9655f141bfd2362b0642 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:56,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:56,335 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:56,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742064_1240 (size=13153) 2024-12-03T06:13:56,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742065_1241 (size=4469) 2024-12-03T06:13:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e34765fe308648faaecf185af6790b81_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206435422/Put/seqid=0 2024-12-03T06:13:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742066_1242 (size=12454) 2024-12-03T06:13:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. as already flushing 2024-12-03T06:13:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206496462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206496463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206496470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206496570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206496570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206496579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206496579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206496580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,755 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#A#compaction#199 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:56,756 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/85a1daecc09f450baeda7d3a11c66392 is 175, key is test_row_0/A:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:56,758 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/965d335019e84518aa733b4e1ff9aa6e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/965d335019e84518aa733b4e1ff9aa6e 2024-12-03T06:13:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742067_1243 (size=32107) 2024-12-03T06:13:56,764 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/B of 87b34dfe780d9655f141bfd2362b0642 into 965d335019e84518aa733b4e1ff9aa6e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:56,764 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:56,764 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/B, priority=13, startTime=1733206436294; duration=0sec 2024-12-03T06:13:56,764 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:13:56,764 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e34765fe308648faaecf185af6790b81_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e34765fe308648faaecf185af6790b81_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:56,764 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:13:56,764 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:13:56,765 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:13:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/6652cf83420b458abaab7c731ed5376d, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:56,765 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 87b34dfe780d9655f141bfd2362b0642/C is initiating minor compaction (all files) 2024-12-03T06:13:56,766 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87b34dfe780d9655f141bfd2362b0642/C in TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:56,766 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6379482aaa15430bb0feb8812d961ba8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp, totalSize=36.7 K 2024-12-03T06:13:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/6652cf83420b458abaab7c731ed5376d is 175, key is test_row_0/A:col10/1733206435422/Put/seqid=0 2024-12-03T06:13:56,767 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6379482aaa15430bb0feb8812d961ba8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733206430916 2024-12-03T06:13:56,768 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a221271be54e4a9082c8fb2c2b3153d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733206432082 2024-12-03T06:13:56,768 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e225ebb9d7274d7cbf5290a4626ac8f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733206434240 2024-12-03T06:13:56,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742068_1244 (size=31255) 2024-12-03T06:13:56,777 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87b34dfe780d9655f141bfd2362b0642#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:13:56,777 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/f9d38b067053443cbc0d8d6228f6b278 is 50, key is test_row_0/C:col10/1733206434283/Put/seqid=0 2024-12-03T06:13:56,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742069_1245 (size=13119) 2024-12-03T06:13:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206496790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206496790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206496790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:56,972 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-12-03T06:13:56,972 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:56,972 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,972 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:56,977 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-12-03T06:13:56,977 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:56,977 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,977 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:56,979 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-12-03T06:13:56,979 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:56,979 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,979 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:56,980 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-12-03T06:13:56,980 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:56,980 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,980 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:56,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206497100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206497100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206497100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,167 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/85a1daecc09f450baeda7d3a11c66392 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/85a1daecc09f450baeda7d3a11c66392 2024-12-03T06:13:57,173 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/A of 87b34dfe780d9655f141bfd2362b0642 into 85a1daecc09f450baeda7d3a11c66392(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:57,173 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/6652cf83420b458abaab7c731ed5376d 2024-12-03T06:13:57,173 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:57,173 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/A, priority=13, startTime=1733206436293; duration=0sec 2024-12-03T06:13:57,173 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:57,173 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:13:57,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/74b5d0c8c6ba45ada2f42583767ab234 is 50, key is test_row_0/B:col10/1733206435422/Put/seqid=0 2024-12-03T06:13:57,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742070_1246 (size=12301) 2024-12-03T06:13:57,196 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/f9d38b067053443cbc0d8d6228f6b278 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/f9d38b067053443cbc0d8d6228f6b278 2024-12-03T06:13:57,201 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87b34dfe780d9655f141bfd2362b0642/C of 87b34dfe780d9655f141bfd2362b0642 into f9d38b067053443cbc0d8d6228f6b278(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:13:57,202 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:57,202 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., storeName=87b34dfe780d9655f141bfd2362b0642/C, priority=13, startTime=1733206436294; duration=0sec 2024-12-03T06:13:57,202 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:13:57,202 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:13:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733206497579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42440 deadline: 1733206497579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,580 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 87b34dfe780d9655f141bfd2362b0642 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:57,581 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in 87b34dfe780d9655f141bfd2362b0642 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,583 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/74b5d0c8c6ba45ada2f42583767ab234 2024-12-03T06:13:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/541ccd56a5e24de8bd84da6e749c489d is 50, key is test_row_0/C:col10/1733206435422/Put/seqid=0 2024-12-03T06:13:57,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742071_1247 (size=12301) 2024-12-03T06:13:57,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:13:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42400 deadline: 1733206497610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42394 deadline: 1733206497610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733206497610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:13:57,998 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/541ccd56a5e24de8bd84da6e749c489d 2024-12-03T06:13:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/6652cf83420b458abaab7c731ed5376d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/6652cf83420b458abaab7c731ed5376d 2024-12-03T06:13:58,010 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/6652cf83420b458abaab7c731ed5376d, entries=150, sequenceid=483, filesize=30.5 K 2024-12-03T06:13:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/74b5d0c8c6ba45ada2f42583767ab234 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/74b5d0c8c6ba45ada2f42583767ab234 2024-12-03T06:13:58,017 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/74b5d0c8c6ba45ada2f42583767ab234, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T06:13:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/541ccd56a5e24de8bd84da6e749c489d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/541ccd56a5e24de8bd84da6e749c489d 2024-12-03T06:13:58,022 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/541ccd56a5e24de8bd84da6e749c489d, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T06:13:58,023 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87b34dfe780d9655f141bfd2362b0642 in 1688ms, sequenceid=483, compaction requested=false 2024-12-03T06:13:58,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:13:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-03T06:13:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-03T06:13:58,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T06:13:58,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-12-03T06:13:58,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8470 sec 2024-12-03T06:13:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T06:13:58,340 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:13:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:58,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:13:58,622 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-12-03T06:13:58,622 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:58,622 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:13:58,622 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:13:58,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:13:58,622 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-12-03T06:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:13:58,622 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:58,622 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,622 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:58,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120338813ba4d25348578bcb4bf76a7d8753_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206438620/Put/seqid=0 2024-12-03T06:13:58,632 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-12-03T06:13:58,632 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:58,632 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,632 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:58,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:58,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742072_1248 (size=12454) 2024-12-03T06:13:59,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:13:59,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120338813ba4d25348578bcb4bf76a7d8753_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120338813ba4d25348578bcb4bf76a7d8753_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:59,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1f05ec634374c5e8fa0df763f0a6d05, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:13:59,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1f05ec634374c5e8fa0df763f0a6d05 is 175, key is test_row_0/A:col10/1733206438620/Put/seqid=0 2024-12-03T06:13:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742073_1249 (size=31255) 2024-12-03T06:13:59,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=505, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1f05ec634374c5e8fa0df763f0a6d05 2024-12-03T06:13:59,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/eea14bd323d244559ff54f4423cb3418 is 50, key is test_row_0/B:col10/1733206438620/Put/seqid=0 2024-12-03T06:13:59,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742074_1250 (size=12301) 2024-12-03T06:13:59,593 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-12-03T06:13:59,593 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:59,593 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,593 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:59,601 INFO [Thread-691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-691. 2024-12-03T06:13:59,601 DEBUG [Thread-691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:13:59,601 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,601 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3527 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3499 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1541 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4622 rows 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1533 2024-12-03T06:13:59,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4595 rows 2024-12-03T06:13:59,602 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:13:59,602 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:13:59,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:13:59,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:13:59,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:13:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:13:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:13:59,607 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206439607"}]},"ts":"1733206439607"} 2024-12-03T06:13:59,609 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:13:59,609 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:13:59,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:13:59,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, UNASSIGN}] 2024-12-03T06:13:59,612 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, UNASSIGN 2024-12-03T06:13:59,613 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:13:59,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, UNASSIGN because future has completed 2024-12-03T06:13:59,615 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:13:59,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:13:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:13:59,768 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:13:59,768 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:13:59,768 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:13:59,768 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 87b34dfe780d9655f141bfd2362b0642, disabling compactions & flushes 2024-12-03T06:13:59,769 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:13:59,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/eea14bd323d244559ff54f4423cb3418 2024-12-03T06:13:59,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/15baea049668474f9874e65d3f18bb29 is 50, key is test_row_0/C:col10/1733206438620/Put/seqid=0 2024-12-03T06:13:59,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742075_1251 (size=12301) 2024-12-03T06:13:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:14:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:14:00,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/15baea049668474f9874e65d3f18bb29 2024-12-03T06:14:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/a1f05ec634374c5e8fa0df763f0a6d05 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1f05ec634374c5e8fa0df763f0a6d05 2024-12-03T06:14:00,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1f05ec634374c5e8fa0df763f0a6d05, entries=150, sequenceid=505, filesize=30.5 K 2024-12-03T06:14:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/eea14bd323d244559ff54f4423cb3418 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/eea14bd323d244559ff54f4423cb3418 2024-12-03T06:14:00,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/eea14bd323d244559ff54f4423cb3418, entries=150, sequenceid=505, filesize=12.0 K 2024-12-03T06:14:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/15baea049668474f9874e65d3f18bb29 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/15baea049668474f9874e65d3f18bb29 2024-12-03T06:14:00,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/15baea049668474f9874e65d3f18bb29, entries=150, sequenceid=505, filesize=12.0 K 2024-12-03T06:14:00,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for 87b34dfe780d9655f141bfd2362b0642 in 1675ms, sequenceid=505, compaction requested=true 2024-12-03T06:14:00,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87b34dfe780d9655f141bfd2362b0642: 2024-12-03T06:14:00,296 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:14:00,296 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:14:00,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:00,296 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. because compaction request was cancelled 2024-12-03T06:14:00,296 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. after waiting 0 ms 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:14:00,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:A 2024-12-03T06:14:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:B, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:00,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. because compaction request was cancelled 2024-12-03T06:14:00,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:B 2024-12-03T06:14:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:00,297 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 87b34dfe780d9655f141bfd2362b0642 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T06:14:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87b34dfe780d9655f141bfd2362b0642:C, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:00,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. because compaction request was cancelled 2024-12-03T06:14:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:00,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87b34dfe780d9655f141bfd2362b0642:C 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=A 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=B 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87b34dfe780d9655f141bfd2362b0642, store=C 2024-12-03T06:14:00,297 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:00,303 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030e8d494d53654b229a3974f3007210ea_87b34dfe780d9655f141bfd2362b0642 is 50, key is test_row_0/A:col10/1733206439591/Put/seqid=0 2024-12-03T06:14:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742076_1252 (size=12454) 2024-12-03T06:14:00,708 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:00,714 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030e8d494d53654b229a3974f3007210ea_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e8d494d53654b229a3974f3007210ea_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:00,715 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/310752f29a954201946ecc6293f6599b, store: [table=TestAcidGuarantees family=A region=87b34dfe780d9655f141bfd2362b0642] 2024-12-03T06:14:00,715 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/310752f29a954201946ecc6293f6599b is 175, key is test_row_0/A:col10/1733206439591/Put/seqid=0 2024-12-03T06:14:00,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742077_1253 (size=31255) 2024-12-03T06:14:00,728 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=511, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/310752f29a954201946ecc6293f6599b 2024-12-03T06:14:00,736 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/329c58fc3de5419a8db0aaa1935ce900 is 50, key is test_row_0/B:col10/1733206439591/Put/seqid=0 2024-12-03T06:14:00,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742078_1254 (size=12301) 2024-12-03T06:14:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:14:01,141 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/329c58fc3de5419a8db0aaa1935ce900 2024-12-03T06:14:01,151 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0ec600102a364a09a6bff864ad9dbdb3 is 50, key is test_row_0/C:col10/1733206439591/Put/seqid=0 2024-12-03T06:14:01,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742079_1255 (size=12301) 2024-12-03T06:14:01,556 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0ec600102a364a09a6bff864ad9dbdb3 2024-12-03T06:14:01,563 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/A/310752f29a954201946ecc6293f6599b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/310752f29a954201946ecc6293f6599b 2024-12-03T06:14:01,569 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/310752f29a954201946ecc6293f6599b, entries=150, sequenceid=511, filesize=30.5 K 2024-12-03T06:14:01,570 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/B/329c58fc3de5419a8db0aaa1935ce900 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/329c58fc3de5419a8db0aaa1935ce900 2024-12-03T06:14:01,573 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/329c58fc3de5419a8db0aaa1935ce900, entries=150, sequenceid=511, filesize=12.0 K 2024-12-03T06:14:01,574 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/.tmp/C/0ec600102a364a09a6bff864ad9dbdb3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0ec600102a364a09a6bff864ad9dbdb3 2024-12-03T06:14:01,579 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0ec600102a364a09a6bff864ad9dbdb3, entries=150, sequenceid=511, filesize=12.0 K 2024-12-03T06:14:01,579 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 87b34dfe780d9655f141bfd2362b0642 in 1282ms, sequenceid=511, compaction requested=true 2024-12-03T06:14:01,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8] to archive 2024-12-03T06:14:01,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:01,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e1e503c2d9de4f0a8acef50b00a0fed5 2024-12-03T06:14:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c5b5f7177d974e0bb134bdcb6f5fab9c 2024-12-03T06:14:01,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/d23bd4a78e804485addf2819e76f81d1 2024-12-03T06:14:01,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/df6afeff21984e05a16d61dd8ae5d2fe 2024-12-03T06:14:01,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/8c51ca277ad04619b53d9982a8555b2c 2024-12-03T06:14:01,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b914ffc0e3c544a3aed7e9287570f44e 2024-12-03T06:14:01,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/94a7593fc5054eeea47828abcf1c30f8 2024-12-03T06:14:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/22b235e585224475907951ece2ece206 2024-12-03T06:14:01,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/e651ccf950ed4fd9be16e7ecf1aa8902 2024-12-03T06:14:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/65962550a7f1457f8635f8986c92e874 2024-12-03T06:14:01,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4fb12820f00b43c594bcefcde17eeabc 2024-12-03T06:14:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/bf13b9a6546c458091f3ec5534f5a53d 2024-12-03T06:14:01,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/b266e72a85684a188174f5ddf60a347e 2024-12-03T06:14:01,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/7f7e10a4319f4f6496ddebfbb647c93d 2024-12-03T06:14:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/ed44c19be2244d1eb6dee2febb179e11 2024-12-03T06:14:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/727e40ccb13b474d9e031b6ea0b489c6 2024-12-03T06:14:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/3bba573b049745e98e47f8f1c4269f61 2024-12-03T06:14:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/fb7e29aacfde4dbe8e1a56dafd36e95e 2024-12-03T06:14:01,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1acbb0fce37467780c0b8f63b7e0628 2024-12-03T06:14:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/4efce3925353443b8306bd1371834c0a 2024-12-03T06:14:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/0198851428f3401d9cea5e567701466d 2024-12-03T06:14:01,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/c8c51ef93b3744eeab433ae7eb15c075 2024-12-03T06:14:01,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/936a0e55be6945e381750136fb7f3f41 2024-12-03T06:14:01,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/04bc445c29354c3da4b7cc72d2857bb7 2024-12-03T06:14:01,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/99404a1e77144b3fa8e59b05c6e126b8 2024-12-03T06:14:01,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/5ba730daf2864576b30187ce4856af16, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/2cd32d69fae54e5b9069b02de0e707ce, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7f9175e0e59743fc94a1bb6d28d15718, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dafd09765d7a410eafba39da2d302826, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/71ab9db7eb6449f9af4fd6daf97005a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/230e632f79a94ea183913c93a0f2749a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/abac5e7778ab416fbec28f97222e1fcd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4] to archive 2024-12-03T06:14:01,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:01,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b63ea0cd1aa44d959287f0f65b295a14 2024-12-03T06:14:01,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/40966d3ad1af473bb278959628ad1e46 2024-12-03T06:14:01,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/9ed38e18905b4094b61f4fe617b7fa91 2024-12-03T06:14:01,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/5ba730daf2864576b30187ce4856af16 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/5ba730daf2864576b30187ce4856af16 2024-12-03T06:14:01,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/b445c67ae620472eb5c112950f324d7d 2024-12-03T06:14:01,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dc58fd6905cc4d5eb845b3aa99a1e924 2024-12-03T06:14:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/2cd32d69fae54e5b9069b02de0e707ce to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/2cd32d69fae54e5b9069b02de0e707ce 2024-12-03T06:14:01,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/fd166eab250049568a00387d13cefdf5 2024-12-03T06:14:01,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/61da4ffc22e746048787bdda782d66d0 2024-12-03T06:14:01,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7f9175e0e59743fc94a1bb6d28d15718 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7f9175e0e59743fc94a1bb6d28d15718 2024-12-03T06:14:01,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/511833d7ab1d47d3806d4e0d7ee3dc88 2024-12-03T06:14:01,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ffceb975db614d01894cb8b862f58a5e 2024-12-03T06:14:01,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dafd09765d7a410eafba39da2d302826 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/dafd09765d7a410eafba39da2d302826 2024-12-03T06:14:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/f4559f4462984530876ae41dcb708f52 2024-12-03T06:14:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/e6cee1c3934f430bbba85e2bd9a4354f 2024-12-03T06:14:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/71ab9db7eb6449f9af4fd6daf97005a0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/71ab9db7eb6449f9af4fd6daf97005a0 2024-12-03T06:14:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/6b003e565a1e44ae8b3fc907e633d70d 2024-12-03T06:14:01,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/537f704c3b71494db3ca94e3b6e66825 2024-12-03T06:14:01,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/230e632f79a94ea183913c93a0f2749a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/230e632f79a94ea183913c93a0f2749a 2024-12-03T06:14:01,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee8d6c9b3daa4014932b89bbddd3f888 2024-12-03T06:14:01,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/975ad8527f57433896329d9418e9623c 2024-12-03T06:14:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/abac5e7778ab416fbec28f97222e1fcd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/abac5e7778ab416fbec28f97222e1fcd 2024-12-03T06:14:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/64fcb60e96bd434b9ce2c7d919e116e2 2024-12-03T06:14:01,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/7ca74c478f1244baa9de5903686857f6 2024-12-03T06:14:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/ee72be29b5664dfa94d6bf3d197668d4 2024-12-03T06:14:01,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9e9ef0f6b1f642e2977d40b2c46d2f2a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1d3568356df04d51ab929d3bae2a8a77, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/098a86da6df24de096925901a6c3c15f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a5f31e29ba304f5aa47551fc8dc88a60, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6379482aaa15430bb0feb8812d961ba8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1] to archive 2024-12-03T06:14:01,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:01,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/cb9f986e400744499ec607984d2035e9 2024-12-03T06:14:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/67f9e500ec0549e3bf318b2f462c86cc 2024-12-03T06:14:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/3914e52030ab49afa029065d1d39c1a4 2024-12-03T06:14:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9e9ef0f6b1f642e2977d40b2c46d2f2a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9e9ef0f6b1f642e2977d40b2c46d2f2a 2024-12-03T06:14:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a8586b9bf9d4418f87eb0b289c6f21f8 2024-12-03T06:14:01,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1c292061b4654f71802981f011ee8c82 2024-12-03T06:14:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1d3568356df04d51ab929d3bae2a8a77 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/1d3568356df04d51ab929d3bae2a8a77 2024-12-03T06:14:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/5f8f1a03b8994ca1962c62afbfa8f5ba 2024-12-03T06:14:01,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0558f23ab2f74c98b8b85f8bed05d191 2024-12-03T06:14:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/4607eb052d5a49948f519285a6ae42e0 2024-12-03T06:14:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/098a86da6df24de096925901a6c3c15f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/098a86da6df24de096925901a6c3c15f 2024-12-03T06:14:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/526c3578684e461b94132cbd08981098 2024-12-03T06:14:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/459016469658465994dd4273660cc8b7 2024-12-03T06:14:01,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d1c437bd19f7482ebf38d59008d86abf 2024-12-03T06:14:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a5f31e29ba304f5aa47551fc8dc88a60 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a5f31e29ba304f5aa47551fc8dc88a60 2024-12-03T06:14:01,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/720f3c2e5ccb45dda44d4de024846773 2024-12-03T06:14:01,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a62ab5cc8c2d4cfbb55d8af73b8c21bc 2024-12-03T06:14:01,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6cffaf301aeb4ed8b7c11f21ac7cb8e9 2024-12-03T06:14:01,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/d7b2d760433140aab21b97764fd6652a 2024-12-03T06:14:01,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/9808c975676540cea88324852c4035f3 2024-12-03T06:14:01,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6379482aaa15430bb0feb8812d961ba8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/6379482aaa15430bb0feb8812d961ba8 2024-12-03T06:14:01,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/be2bc4938074478a9771691f3ec9beeb 2024-12-03T06:14:01,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/a221271be54e4a9082c8fb2c2b3153d6 2024-12-03T06:14:01,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/e225ebb9d7274d7cbf5290a4626ac8f1 2024-12-03T06:14:01,680 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits/514.seqid, newMaxSeqId=514, maxSeqId=4 2024-12-03T06:14:01,681 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. 2024-12-03T06:14:01,681 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 87b34dfe780d9655f141bfd2362b0642: Waiting for close lock at 1733206439768Running coprocessor pre-close hooks at 1733206439768Disabling compacts and flushes for region at 1733206439768Disabling writes for close at 1733206440296 (+528 ms)Obtaining lock to block concurrent updates at 1733206440297 (+1 ms)Preparing flush snapshotting stores in 87b34dfe780d9655f141bfd2362b0642 at 1733206440297Finished memstore snapshotting TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733206440297Flushing stores of TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642. at 1733206440297Flushing 87b34dfe780d9655f141bfd2362b0642/A: creating writer at 1733206440298 (+1 ms)Flushing mob file 87b34dfe780d9655f141bfd2362b0642/A: appending metadata at 1733206440303 (+5 ms)Flushing mob file 87b34dfe780d9655f141bfd2362b0642/A: closing flushed file at 1733206440303Flushing 87b34dfe780d9655f141bfd2362b0642/A: appending metadata at 1733206440715 (+412 ms)Flushing 87b34dfe780d9655f141bfd2362b0642/A: closing flushed file at 1733206440715Flushing 87b34dfe780d9655f141bfd2362b0642/B: creating writer at 1733206440732 (+17 ms)Flushing 87b34dfe780d9655f141bfd2362b0642/B: appending metadata at 1733206440735 (+3 ms)Flushing 87b34dfe780d9655f141bfd2362b0642/B: closing flushed file at 1733206440735Flushing 87b34dfe780d9655f141bfd2362b0642/C: creating writer at 1733206441147 (+412 ms)Flushing 87b34dfe780d9655f141bfd2362b0642/C: appending metadata at 1733206441150 (+3 ms)Flushing 87b34dfe780d9655f141bfd2362b0642/C: closing flushed file at 1733206441151 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f6b5ae7: reopening flushed file at 1733206441562 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b8f1b7a: reopening flushed file at 1733206441569 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f141f9e: reopening flushed file at 1733206441573 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 87b34dfe780d9655f141bfd2362b0642 in 1282ms, sequenceid=511, compaction requested=true at 1733206441579 (+6 ms)Writing region close event to WAL at 1733206441676 (+97 ms)Running coprocessor post-close hooks at 1733206441681 (+5 ms)Closed at 1733206441681 2024-12-03T06:14:01,683 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=87b34dfe780d9655f141bfd2362b0642, regionState=CLOSED 2024-12-03T06:14:01,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:01,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T06:14:01,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 87b34dfe780d9655f141bfd2362b0642, server=122cddc95ec8,35255,1733206381424 in 2.0710 sec 2024-12-03T06:14:01,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-03T06:14:01,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87b34dfe780d9655f141bfd2362b0642, UNASSIGN in 2.0750 sec 2024-12-03T06:14:01,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T06:14:01,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0800 sec 2024-12-03T06:14:01,693 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206441693"}]},"ts":"1733206441693"} 2024-12-03T06:14:01,695 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:14:01,695 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:14:01,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0930 sec 2024-12-03T06:14:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T06:14:01,750 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:01,750 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:14:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,752 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T06:14:01,753 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,759 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits] 2024-12-03T06:14:01,762 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/310752f29a954201946ecc6293f6599b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/310752f29a954201946ecc6293f6599b 2024-12-03T06:14:01,764 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/6652cf83420b458abaab7c731ed5376d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/6652cf83420b458abaab7c731ed5376d 2024-12-03T06:14:01,765 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/85a1daecc09f450baeda7d3a11c66392 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/85a1daecc09f450baeda7d3a11c66392 2024-12-03T06:14:01,766 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1f05ec634374c5e8fa0df763f0a6d05 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/A/a1f05ec634374c5e8fa0df763f0a6d05 2024-12-03T06:14:01,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/329c58fc3de5419a8db0aaa1935ce900 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/329c58fc3de5419a8db0aaa1935ce900 2024-12-03T06:14:01,769 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/74b5d0c8c6ba45ada2f42583767ab234 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/74b5d0c8c6ba45ada2f42583767ab234 2024-12-03T06:14:01,770 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/965d335019e84518aa733b4e1ff9aa6e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/965d335019e84518aa733b4e1ff9aa6e 2024-12-03T06:14:01,771 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/eea14bd323d244559ff54f4423cb3418 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/B/eea14bd323d244559ff54f4423cb3418 2024-12-03T06:14:01,773 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0ec600102a364a09a6bff864ad9dbdb3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/0ec600102a364a09a6bff864ad9dbdb3 2024-12-03T06:14:01,774 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/15baea049668474f9874e65d3f18bb29 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/15baea049668474f9874e65d3f18bb29 2024-12-03T06:14:01,775 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/541ccd56a5e24de8bd84da6e749c489d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/541ccd56a5e24de8bd84da6e749c489d 2024-12-03T06:14:01,776 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/f9d38b067053443cbc0d8d6228f6b278 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/C/f9d38b067053443cbc0d8d6228f6b278 2024-12-03T06:14:01,779 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits/514.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642/recovered.edits/514.seqid 2024-12-03T06:14:01,780 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,780 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:14:01,781 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:14:01,781 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T06:14:01,785 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203038a6bb6a0314b699aea4df2735c5663_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203038a6bb6a0314b699aea4df2735c5663_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,786 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e8d494d53654b229a3974f3007210ea_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e8d494d53654b229a3974f3007210ea_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,787 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314b8c496fe9a41b0ad9c3d3372f54a5e_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314b8c496fe9a41b0ad9c3d3372f54a5e_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,788 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120319c9cde737a94c71baee0134730bb3be_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120319c9cde737a94c71baee0134730bb3be_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,790 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f419d7bdebd4837b14ebdd2cae2fbba_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f419d7bdebd4837b14ebdd2cae2fbba_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,791 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032082e353d3b54603aff07c190a8cee84_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032082e353d3b54603aff07c190a8cee84_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,792 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120325eaaca9686f46f7ad5b453ef29190db_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120325eaaca9686f46f7ad5b453ef29190db_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,793 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120338813ba4d25348578bcb4bf76a7d8753_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120338813ba4d25348578bcb4bf76a7d8753_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,794 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033976ed3c3b334fae80ff94181c2104e1_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033976ed3c3b334fae80ff94181c2104e1_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,795 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120344e648bf6ef444948b27f73681828c90_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120344e648bf6ef444948b27f73681828c90_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,796 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203489db116c5704463a74b0dd1675ef3c3_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203489db116c5704463a74b0dd1675ef3c3_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,797 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120359805097a3f64e7b9105f6ba9e3cc23e_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120359805097a3f64e7b9105f6ba9e3cc23e_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,798 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a9593dbcc174b22968cfaf3ef0ad11c_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a9593dbcc174b22968cfaf3ef0ad11c_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,799 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fc84b393d6f4d8aa3fdb7bc34a6f4a2_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fc84b393d6f4d8aa3fdb7bc34a6f4a2_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,800 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037be7498fcbcf47ce8e35baea5000a013_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037be7498fcbcf47ce8e35baea5000a013_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,802 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ddfcedabb1b4aba8106059561bad3cc_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ddfcedabb1b4aba8106059561bad3cc_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,803 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b779313a43a04f538cf074a3242a6336_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b779313a43a04f538cf074a3242a6336_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,804 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc6c85446cbd4de8a6e37e3e0ba9715a_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc6c85446cbd4de8a6e37e3e0ba9715a_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,805 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c41c8ca95840440b8a3d2ae92cc6cb43_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c41c8ca95840440b8a3d2ae92cc6cb43_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,806 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1afd638049f41d7a124a5ae1a379ffb_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1afd638049f41d7a124a5ae1a379ffb_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,807 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e34765fe308648faaecf185af6790b81_87b34dfe780d9655f141bfd2362b0642 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e34765fe308648faaecf185af6790b81_87b34dfe780d9655f141bfd2362b0642 2024-12-03T06:14:01,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:14:01,810 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,813 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:14:01,815 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:14:01,817 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,817 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:14:01,817 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206441817"}]},"ts":"9223372036854775807"} 2024-12-03T06:14:01,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:14:01,820 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 87b34dfe780d9655f141bfd2362b0642, NAME => 'TestAcidGuarantees,,1733206413746.87b34dfe780d9655f141bfd2362b0642.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:14:01,820 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:14:01,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206441820"}]},"ts":"9223372036854775807"} 2024-12-03T06:14:01,822 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:14:01,822 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-12-03T06:14:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T06:14:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:14:01,860 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:01,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-161939200_22 at /127.0.0.1:41122 [Waiting for operation #1119] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1228833144_22 at /127.0.0.1:40846 [Waiting for operation #1246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1228833144_22 at /127.0.0.1:46282 [Waiting for operation #1514] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-161939200_22 at /127.0.0.1:57366 [Waiting for operation #1574] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 442) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=12042 (was 12797) 2024-12-03T06:14:01,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=12041 2024-12-03T06:14:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:14:01,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:14:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:01,885 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:14:01,885 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:01,885 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-12-03T06:14:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T06:14:01,886 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:14:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742080_1256 (size=1061) 2024-12-03T06:14:01,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:14:01,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742081_1257 (size=53) 2024-12-03T06:14:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T06:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5535461f8d6a19172fe21e7d7b6b77aa, disabling compactions & flushes 2024-12-03T06:14:02,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. after waiting 0 ms 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5535461f8d6a19172fe21e7d7b6b77aa: Waiting for close lock at 1733206442300Disabling compacts and flushes for region at 1733206442300Disabling writes for close at 1733206442300Writing region close event to WAL at 1733206442300Closed at 1733206442300 2024-12-03T06:14:02,301 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:14:02,301 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206442301"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206442301"}]},"ts":"1733206442301"} 2024-12-03T06:14:02,303 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:14:02,304 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:14:02,304 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206442304"}]},"ts":"1733206442304"} 2024-12-03T06:14:02,305 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:14:02,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, ASSIGN}] 2024-12-03T06:14:02,306 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, ASSIGN 2024-12-03T06:14:02,307 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:14:02,458 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5535461f8d6a19172fe21e7d7b6b77aa, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:02,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, ASSIGN because future has completed 2024-12-03T06:14:02,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:14:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T06:14:02,618 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,618 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:14:02,618 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,618 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:14:02,619 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,619 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,620 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,621 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:02,621 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5535461f8d6a19172fe21e7d7b6b77aa columnFamilyName A 2024-12-03T06:14:02,621 DEBUG [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:02,621 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(327): Store=5535461f8d6a19172fe21e7d7b6b77aa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:02,622 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,623 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:02,623 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5535461f8d6a19172fe21e7d7b6b77aa columnFamilyName B 2024-12-03T06:14:02,623 DEBUG [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:02,623 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(327): Store=5535461f8d6a19172fe21e7d7b6b77aa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:02,623 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,624 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:02,625 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5535461f8d6a19172fe21e7d7b6b77aa columnFamilyName C 2024-12-03T06:14:02,625 DEBUG [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:02,625 INFO [StoreOpener-5535461f8d6a19172fe21e7d7b6b77aa-1 {}] regionserver.HStore(327): Store=5535461f8d6a19172fe21e7d7b6b77aa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:02,625 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,625 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,626 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,627 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,628 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,628 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,628 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:14:02,629 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,632 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:14:02,632 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 5535461f8d6a19172fe21e7d7b6b77aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71586316, jitterRate=0.06671923398971558}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:14:02,632 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:02,633 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 5535461f8d6a19172fe21e7d7b6b77aa: Running coprocessor pre-open hook at 1733206442619Writing region info on filesystem at 1733206442619Initializing all the Stores at 1733206442619Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206442620 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206442620Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206442620Cleaning up temporary data from old regions at 1733206442628 (+8 ms)Running coprocessor post-open hooks at 1733206442632 (+4 ms)Region opened successfully at 1733206442632 2024-12-03T06:14:02,633 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., pid=69, masterSystemTime=1733206442614 2024-12-03T06:14:02,635 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,635 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:02,636 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5535461f8d6a19172fe21e7d7b6b77aa, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:02,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:02,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T06:14:02,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 in 177 msec 2024-12-03T06:14:02,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-12-03T06:14:02,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, ASSIGN in 334 msec 2024-12-03T06:14:02,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:14:02,642 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206442642"}]},"ts":"1733206442642"} 2024-12-03T06:14:02,644 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:14:02,645 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:14:02,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 762 msec 2024-12-03T06:14:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T06:14:03,020 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:03,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,024 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4383b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbd7641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:14:03,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:14:03,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d53ccca 2024-12-03T06:14:03,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:14:03,033 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:14:03,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2c2fda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd9773c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,040 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25adb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7754c38b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299a38f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,046 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1219966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,049 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c50be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b723e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50840240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,056 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a774bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4099af63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d8eebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,061 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bbbb8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b00c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac1f30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c3d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,068 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f8cddbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@114bdb10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:03,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:03,072 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1adc65cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:03,073 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f397a01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:03,074 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,074 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,074 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,074 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,075 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,075 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,076 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T06:14:03,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:03,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:03,079 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,080 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,083 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:03,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:14:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:03,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:14:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:03,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733206443113, max timestamp=1733206443123Num uniques 859; 2024-12-03T06:14:03,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:14:03,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206443113, max timestamp=1733206443124Num uniques 859; 2024-12-03T06:14:03,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:14:03,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206443113, max timestamp=1733206443124Num uniques 859; 2024-12-03T06:14:03,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:14:03,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b25c5ee99f604c39a8ee27ffb5db2284 is 50, key is test_row_0/A:col10/1733206443095/Put/seqid=0 2024-12-03T06:14:03,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742082_1258 (size=12001) 2024-12-03T06:14:03,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b25c5ee99f604c39a8ee27ffb5db2284 2024-12-03T06:14:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206503148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206503148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206503150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206503152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206503154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4e5a755ba3354251889c05d6c1510387 is 50, key is test_row_0/B:col10/1733206443095/Put/seqid=0 2024-12-03T06:14:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742083_1259 (size=12001) 2024-12-03T06:14:03,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206503259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206503259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206503260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206503260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206503270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206503469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206503469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206503469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206503469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206503479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:03,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:03,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4e5a755ba3354251889c05d6c1510387 2024-12-03T06:14:03,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/60cc98d4536c416e80ba9f99cef45641 is 50, key is test_row_0/C:col10/1733206443095/Put/seqid=0 2024-12-03T06:14:03,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742084_1260 (size=12001) 2024-12-03T06:14:03,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:03,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206503779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206503779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206503779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206503780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206503790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:03,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:03,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:03,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:03,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:04,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:04,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:04,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:04,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:04,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:04,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:04,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/60cc98d4536c416e80ba9f99cef45641 2024-12-03T06:14:04,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b25c5ee99f604c39a8ee27ffb5db2284 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284 2024-12-03T06:14:04,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:14:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4e5a755ba3354251889c05d6c1510387 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387 2024-12-03T06:14:04,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:14:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/60cc98d4536c416e80ba9f99cef45641 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641 2024-12-03T06:14:04,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:14:04,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5535461f8d6a19172fe21e7d7b6b77aa in 1013ms, sequenceid=12, compaction requested=false 2024-12-03T06:14:04,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:04,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:04,164 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/2111573a0cae4152aa1d8bd5be8310cc is 50, key is test_row_0/A:col10/1733206443152/Put/seqid=0 2024-12-03T06:14:04,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742085_1261 (size=12001) 2024-12-03T06:14:04,205 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/2111573a0cae4152aa1d8bd5be8310cc 2024-12-03T06:14:04,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/e27587f66ae84b12b3c9383cdf78c862 is 50, key is test_row_0/B:col10/1733206443152/Put/seqid=0 2024-12-03T06:14:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:04,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742086_1262 (size=12001) 2024-12-03T06:14:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:04,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206504299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206504299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206504299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206504300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206504300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206504620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206504620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206504621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206504621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,633 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/e27587f66ae84b12b3c9383cdf78c862 2024-12-03T06:14:04,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/a6bd9ac95725443db4efb4001b3879a2 is 50, key is test_row_0/C:col10/1733206443152/Put/seqid=0 2024-12-03T06:14:04,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742087_1263 (size=12001) 2024-12-03T06:14:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,076 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/a6bd9ac95725443db4efb4001b3879a2 2024-12-03T06:14:05,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/2111573a0cae4152aa1d8bd5be8310cc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc 2024-12-03T06:14:05,086 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T06:14:05,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/e27587f66ae84b12b3c9383cdf78c862 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862 2024-12-03T06:14:05,092 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T06:14:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/a6bd9ac95725443db4efb4001b3879a2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2 2024-12-03T06:14:05,098 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T06:14:05,100 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 5535461f8d6a19172fe21e7d7b6b77aa in 936ms, sequenceid=47, compaction requested=false 2024-12-03T06:14:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T06:14:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T06:14:05,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T06:14:05,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0150 sec 2024-12-03T06:14:05,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0310 sec 2024-12-03T06:14:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T06:14:05,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:05,230 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T06:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:05,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:05,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:05,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:05,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:05,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/248a681ab2ba42b59c9232101b0b3c14 is 50, key is test_row_0/A:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742088_1264 (size=14341) 2024-12-03T06:14:05,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/248a681ab2ba42b59c9232101b0b3c14 2024-12-03T06:14:05,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/ef19698c1d5f4e4682693da4df3e45e8 is 50, key is test_row_0/B:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:05,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742089_1265 (size=12001) 2024-12-03T06:14:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206505382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206505439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206505439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206505439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206505439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206505490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:05,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:05,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:05,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:05,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206505699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:05,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/ef19698c1d5f4e4682693da4df3e45e8 2024-12-03T06:14:05,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/87e3268d840146cb8565f2d6769c2c93 is 50, key is test_row_0/C:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742090_1266 (size=12001) 2024-12-03T06:14:05,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:05,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:05,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:06,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206506010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:06,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/87e3268d840146cb8565f2d6769c2c93 2024-12-03T06:14:06,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/248a681ab2ba42b59c9232101b0b3c14 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14 2024-12-03T06:14:06,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14, entries=200, sequenceid=59, filesize=14.0 K 2024-12-03T06:14:06,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/ef19698c1d5f4e4682693da4df3e45e8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8 2024-12-03T06:14:06,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T06:14:06,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/87e3268d840146cb8565f2d6769c2c93 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93 2024-12-03T06:14:06,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T06:14:06,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5535461f8d6a19172fe21e7d7b6b77aa in 886ms, sequenceid=59, compaction requested=true 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:06,198 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:06,198 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:06,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:06,199 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:06,199 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:06,199 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:06,199 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:06,199 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,199 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,200 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.2 K 2024-12-03T06:14:06,200 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=37.4 K 2024-12-03T06:14:06,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25c5ee99f604c39a8ee27ffb5db2284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206443086 2024-12-03T06:14:06,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2111573a0cae4152aa1d8bd5be8310cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733206443148 2024-12-03T06:14:06,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 248a681ab2ba42b59c9232101b0b3c14, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:06,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5a755ba3354251889c05d6c1510387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206443086 2024-12-03T06:14:06,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e27587f66ae84b12b3c9383cdf78c862, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733206443148 2024-12-03T06:14:06,202 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ef19698c1d5f4e4682693da4df3e45e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:06,210 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:06,210 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:06,210 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/2318896b52fa4a37a643a4b242c0d116 is 50, key is test_row_0/B:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:06,210 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/d1738c6b6eb04ee5a8047f299e4b481e is 50, key is test_row_0/A:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742092_1268 (size=12104) 2024-12-03T06:14:06,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742091_1267 (size=12104) 2024-12-03T06:14:06,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T06:14:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,310 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:14:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:06,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e4373b1cfb144205aac933312e082477 is 50, key is test_row_0/A:col10/1733206445375/Put/seqid=0 2024-12-03T06:14:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742093_1269 (size=12001) 2024-12-03T06:14:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:06,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206506461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206506463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206506466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206506466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206506519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206506580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206506580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206506580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206506580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,636 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/d1738c6b6eb04ee5a8047f299e4b481e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/d1738c6b6eb04ee5a8047f299e4b481e 2024-12-03T06:14:06,641 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into d1738c6b6eb04ee5a8047f299e4b481e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:06,641 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:06,641 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=13, startTime=1733206446198; duration=0sec 2024-12-03T06:14:06,641 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:06,641 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:06,641 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:06,642 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:06,642 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:06,642 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:06,642 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.2 K 2024-12-03T06:14:06,643 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60cc98d4536c416e80ba9f99cef45641, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206443086 2024-12-03T06:14:06,643 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bd9ac95725443db4efb4001b3879a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733206443148 2024-12-03T06:14:06,643 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e3268d840146cb8565f2d6769c2c93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:06,649 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:06,650 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/3fc6c61d65c345b680926d1eb52fb6e4 is 50, key is test_row_0/C:col10/1733206445311/Put/seqid=0 2024-12-03T06:14:06,654 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/2318896b52fa4a37a643a4b242c0d116 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/2318896b52fa4a37a643a4b242c0d116 2024-12-03T06:14:06,658 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into 2318896b52fa4a37a643a4b242c0d116(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:06,659 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:06,659 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=13, startTime=1733206446198; duration=0sec 2024-12-03T06:14:06,659 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:06,659 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:06,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742094_1270 (size=12104) 2024-12-03T06:14:06,728 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e4373b1cfb144205aac933312e082477 2024-12-03T06:14:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9e654c0ba86c40f999b135ec80dfa419 is 50, key is test_row_0/B:col10/1733206445375/Put/seqid=0 2024-12-03T06:14:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742095_1271 (size=12001) 2024-12-03T06:14:06,744 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9e654c0ba86c40f999b135ec80dfa419 2024-12-03T06:14:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b447cacdfe4740e392008139b2a4f88a is 50, key is test_row_0/C:col10/1733206445375/Put/seqid=0 2024-12-03T06:14:06,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742096_1272 (size=12001) 2024-12-03T06:14:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206506790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206506790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206506790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206506790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,088 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/3fc6c61d65c345b680926d1eb52fb6e4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/3fc6c61d65c345b680926d1eb52fb6e4 2024-12-03T06:14:07,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into 3fc6c61d65c345b680926d1eb52fb6e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:07,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:07,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=13, startTime=1733206446198; duration=0sec 2024-12-03T06:14:07,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:07,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206507099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206507100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206507100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206507100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,168 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b447cacdfe4740e392008139b2a4f88a 2024-12-03T06:14:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e4373b1cfb144205aac933312e082477 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477 2024-12-03T06:14:07,176 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477, entries=150, sequenceid=79, filesize=11.7 K 2024-12-03T06:14:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9e654c0ba86c40f999b135ec80dfa419 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419 2024-12-03T06:14:07,181 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419, entries=150, sequenceid=79, filesize=11.7 K 2024-12-03T06:14:07,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b447cacdfe4740e392008139b2a4f88a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a 2024-12-03T06:14:07,188 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a, entries=150, sequenceid=79, filesize=11.7 K 2024-12-03T06:14:07,189 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5535461f8d6a19172fe21e7d7b6b77aa in 879ms, sequenceid=79, compaction requested=false 2024-12-03T06:14:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T06:14:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T06:14:07,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T06:14:07,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-12-03T06:14:07,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9640 sec 2024-12-03T06:14:07,213 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:14:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T06:14:07,381 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:07,381 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T06:14:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T06:14:07,383 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:07,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:07,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T06:14:07,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:07,538 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:07,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:07,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/c8e6fe3e8cec4bcb98137b91fe240884 is 50, key is test_row_0/A:col10/1733206446464/Put/seqid=0 2024-12-03T06:14:07,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742097_1273 (size=12001) 2024-12-03T06:14:07,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206507608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206507610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206507610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206507610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206507610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T06:14:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206507720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206507929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:07,970 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/c8e6fe3e8cec4bcb98137b91fe240884 2024-12-03T06:14:07,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/a07e94a5bf394cb7b51960cfb8aa5451 is 50, key is test_row_0/B:col10/1733206446464/Put/seqid=0 2024-12-03T06:14:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T06:14:08,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742098_1274 (size=12001) 2024-12-03T06:14:08,033 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/a07e94a5bf394cb7b51960cfb8aa5451 2024-12-03T06:14:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c30c64a3fff149b3ac8fe880ea0aeede is 50, key is test_row_0/C:col10/1733206446464/Put/seqid=0 2024-12-03T06:14:08,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742099_1275 (size=12001) 2024-12-03T06:14:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206508239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,464 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c30c64a3fff149b3ac8fe880ea0aeede 2024-12-03T06:14:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/c8e6fe3e8cec4bcb98137b91fe240884 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884 2024-12-03T06:14:08,479 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T06:14:08,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/a07e94a5bf394cb7b51960cfb8aa5451 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451 2024-12-03T06:14:08,489 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T06:14:08,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c30c64a3fff149b3ac8fe880ea0aeede as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede 2024-12-03T06:14:08,497 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T06:14:08,499 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5535461f8d6a19172fe21e7d7b6b77aa in 960ms, sequenceid=94, compaction requested=true 2024-12-03T06:14:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T06:14:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T06:14:08,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T06:14:08,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1150 sec 2024-12-03T06:14:08,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1230 sec 2024-12-03T06:14:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T06:14:08,520 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:08,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-03T06:14:08,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:08,524 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:08,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:08,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T06:14:08,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206447596, max timestamp=1733206448621Num uniques 859; 2024-12-03T06:14:08,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206447596, max timestamp=1733206448621Num uniques 859; 2024-12-03T06:14:08,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:08,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:08,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/3a04eb171b394d5c9bdc47a24bbcb5e2 is 50, key is test_row_0/A:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742100_1276 (size=14341) 2024-12-03T06:14:08,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T06:14:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206508667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:08,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/3a04eb171b394d5c9bdc47a24bbcb5e2 2024-12-03T06:14:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206508669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206508669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206508672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/efff96ec27fa4aab8fd9caf2a95a6354 is 50, key is test_row_0/B:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742101_1277 (size=12001) 2024-12-03T06:14:08,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206508750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206508790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206508790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206508790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206508790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:08,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T06:14:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:08,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T06:14:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:08,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:09,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206509000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206509001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206509001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206509001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/efff96ec27fa4aab8fd9caf2a95a6354 2024-12-03T06:14:09,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/5cfbacbd18e447838659b6cae29dd9f3 is 50, key is test_row_0/C:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:09,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T06:14:09,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:09,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:09,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:09,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742102_1278 (size=12001) 2024-12-03T06:14:09,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/5cfbacbd18e447838659b6cae29dd9f3 2024-12-03T06:14:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/3a04eb171b394d5c9bdc47a24bbcb5e2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2 2024-12-03T06:14:09,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2, entries=200, sequenceid=116, filesize=14.0 K 2024-12-03T06:14:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/efff96ec27fa4aab8fd9caf2a95a6354 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354 2024-12-03T06:14:09,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354, entries=150, sequenceid=116, filesize=11.7 K 2024-12-03T06:14:09,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/5cfbacbd18e447838659b6cae29dd9f3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3 2024-12-03T06:14:09,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3, entries=150, sequenceid=116, filesize=11.7 K 2024-12-03T06:14:09,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 5535461f8d6a19172fe21e7d7b6b77aa in 564ms, sequenceid=116, compaction requested=true 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:09,187 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:14:09,187 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:09,190 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:09,190 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:09,190 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:09,190 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:09,190 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,190 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/2318896b52fa4a37a643a4b242c0d116, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=47.0 K 2024-12-03T06:14:09,190 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,190 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/d1738c6b6eb04ee5a8047f299e4b481e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=49.3 K 2024-12-03T06:14:09,191 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2318896b52fa4a37a643a4b242c0d116, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:09,192 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1738c6b6eb04ee5a8047f299e4b481e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:09,192 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e654c0ba86c40f999b135ec80dfa419, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733206445363 2024-12-03T06:14:09,192 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4373b1cfb144205aac933312e082477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733206445363 2024-12-03T06:14:09,192 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a07e94a5bf394cb7b51960cfb8aa5451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733206446450 2024-12-03T06:14:09,192 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e6fe3e8cec4bcb98137b91fe240884, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733206446450 2024-12-03T06:14:09,193 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting efff96ec27fa4aab8fd9caf2a95a6354, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:09,193 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a04eb171b394d5c9bdc47a24bbcb5e2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:09,204 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:09,204 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/082a02381ceb43f098051d6199693545 is 50, key is test_row_0/B:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:09,205 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:09,205 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/a6c13525f3c54bbdb50c847cc36e556d is 50, key is test_row_0/A:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:09,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742104_1280 (size=12241) 2024-12-03T06:14:09,231 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/a6c13525f3c54bbdb50c847cc36e556d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a6c13525f3c54bbdb50c847cc36e556d 2024-12-03T06:14:09,243 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into a6c13525f3c54bbdb50c847cc36e556d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:09,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:09,243 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=12, startTime=1733206449187; duration=0sec 2024-12-03T06:14:09,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:09,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:09,243 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:09,245 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:09,245 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:09,245 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,245 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/3fc6c61d65c345b680926d1eb52fb6e4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=47.0 K 2024-12-03T06:14:09,246 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc6c61d65c345b680926d1eb52fb6e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733206444297 2024-12-03T06:14:09,247 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b447cacdfe4740e392008139b2a4f88a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733206445363 2024-12-03T06:14:09,247 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30c64a3fff149b3ac8fe880ea0aeede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733206446450 2024-12-03T06:14:09,247 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfbacbd18e447838659b6cae29dd9f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:09,266 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#234 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:09,267 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/f2991f24616e4cdaab300185fa2a4280 is 50, key is test_row_0/C:col10/1733206447596/Put/seqid=0 2024-12-03T06:14:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742103_1279 (size=12241) 2024-12-03T06:14:09,278 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/082a02381ceb43f098051d6199693545 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/082a02381ceb43f098051d6199693545 2024-12-03T06:14:09,285 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into 082a02381ceb43f098051d6199693545(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:09,285 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:09,285 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=12, startTime=1733206449187; duration=0sec 2024-12-03T06:14:09,285 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:09,285 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:09,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,298 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742105_1281 (size=12241) 2024-12-03T06:14:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e6a07e15f7dc4e37850b025d3de8aeaa is 50, key is test_row_0/A:col10/1733206448666/Put/seqid=0 2024-12-03T06:14:09,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:09,316 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/f2991f24616e4cdaab300185fa2a4280 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f2991f24616e4cdaab300185fa2a4280 2024-12-03T06:14:09,322 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into f2991f24616e4cdaab300185fa2a4280(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:09,322 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:09,322 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=12, startTime=1733206449187; duration=0sec 2024-12-03T06:14:09,322 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:09,322 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742106_1282 (size=12151) 2024-12-03T06:14:09,325 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e6a07e15f7dc4e37850b025d3de8aeaa 2024-12-03T06:14:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206509326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206509328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206509332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206509334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/57e2ab634ba04f6396248c142594ad1d is 50, key is test_row_0/B:col10/1733206448666/Put/seqid=0 2024-12-03T06:14:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742107_1283 (size=12151) 2024-12-03T06:14:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206509440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206509440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206509440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206509440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206509650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206509650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206509650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206509650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:09,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206509770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:09,771 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 5535461f8d6a19172fe21e7d7b6b77aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:09,783 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/57e2ab634ba04f6396248c142594ad1d 2024-12-03T06:14:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/bb9d8700e4784cf79ec2e93d129c9cd2 is 50, key is test_row_0/C:col10/1733206448666/Put/seqid=0 2024-12-03T06:14:09,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742108_1284 (size=12151) 2024-12-03T06:14:09,801 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/bb9d8700e4784cf79ec2e93d129c9cd2 2024-12-03T06:14:09,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e6a07e15f7dc4e37850b025d3de8aeaa as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa 2024-12-03T06:14:09,816 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T06:14:09,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/57e2ab634ba04f6396248c142594ad1d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d 2024-12-03T06:14:09,823 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T06:14:09,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/bb9d8700e4784cf79ec2e93d129c9cd2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2 2024-12-03T06:14:09,835 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T06:14:09,836 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5535461f8d6a19172fe21e7d7b6b77aa in 538ms, sequenceid=138, compaction requested=false 2024-12-03T06:14:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-03T06:14:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-03T06:14:09,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T06:14:09,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-12-03T06:14:09,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3230 sec 2024-12-03T06:14:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:09,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:09,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bac38f67555a41fb9d2bd0210873db6d is 50, key is test_row_0/A:col10/1733206449325/Put/seqid=0 2024-12-03T06:14:09,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742109_1285 (size=12151) 2024-12-03T06:14:09,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bac38f67555a41fb9d2bd0210873db6d 2024-12-03T06:14:09,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/c288044439a549adadecd6b397d2f044 is 50, key is test_row_0/B:col10/1733206449325/Put/seqid=0 2024-12-03T06:14:10,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206509996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206509997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206509999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206510002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742110_1286 (size=12151) 2024-12-03T06:14:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206510110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206510110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206510110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206510110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206510320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206510320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206510320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206510320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/c288044439a549adadecd6b397d2f044 2024-12-03T06:14:10,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/9d1cf5ba797c4e6294209e4bafe37f1b is 50, key is test_row_0/C:col10/1733206449325/Put/seqid=0 2024-12-03T06:14:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742111_1287 (size=12151) 2024-12-03T06:14:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206510630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206510630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206510630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206510630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T06:14:10,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:10,670 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-03T06:14:10,672 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T06:14:10,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:10,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T06:14:10,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T06:14:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:10,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:10,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/9d1cf5ba797c4e6294209e4bafe37f1b 2024-12-03T06:14:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bac38f67555a41fb9d2bd0210873db6d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d 2024-12-03T06:14:10,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T06:14:10,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/c288044439a549adadecd6b397d2f044 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044 2024-12-03T06:14:10,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T06:14:10,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/9d1cf5ba797c4e6294209e4bafe37f1b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b 2024-12-03T06:14:10,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T06:14:10,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5535461f8d6a19172fe21e7d7b6b77aa in 898ms, sequenceid=153, compaction requested=true 2024-12-03T06:14:10,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:10,860 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:10,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:10,861 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:10,866 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:10,866 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:10,866 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:10,866 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:10,866 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a6c13525f3c54bbdb50c847cc36e556d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.7 K 2024-12-03T06:14:10,866 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:10,867 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:10,867 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/082a02381ceb43f098051d6199693545, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.7 K 2024-12-03T06:14:10,867 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c13525f3c54bbdb50c847cc36e556d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:10,867 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 082a02381ceb43f098051d6199693545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:10,867 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6a07e15f7dc4e37850b025d3de8aeaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206448656 2024-12-03T06:14:10,868 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e2ab634ba04f6396248c142594ad1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206448656 2024-12-03T06:14:10,868 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bac38f67555a41fb9d2bd0210873db6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733206449325 2024-12-03T06:14:10,869 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c288044439a549adadecd6b397d2f044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733206449325 2024-12-03T06:14:10,882 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:10,883 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b37e3e5d8ef847879ebbe41f31a78d79 is 50, key is test_row_0/A:col10/1733206449325/Put/seqid=0 2024-12-03T06:14:10,883 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:10,884 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/04e0e9b731174784b2991ae55d1439bc is 50, key is test_row_0/B:col10/1733206449325/Put/seqid=0 2024-12-03T06:14:10,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742112_1288 (size=12493) 2024-12-03T06:14:10,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742113_1289 (size=12493) 2024-12-03T06:14:10,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T06:14:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:10,980 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T06:14:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bce2ede5b77d405882bc85464a2c339b is 50, key is test_row_0/A:col10/1733206449995/Put/seqid=0 2024-12-03T06:14:11,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742114_1290 (size=12151) 2024-12-03T06:14:11,053 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bce2ede5b77d405882bc85464a2c339b 2024-12-03T06:14:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/8f376ac67b6a405bb7eb8c1e0e62ea62 is 50, key is test_row_0/B:col10/1733206449995/Put/seqid=0 2024-12-03T06:14:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742115_1291 (size=12151) 2024-12-03T06:14:11,100 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/8f376ac67b6a405bb7eb8c1e0e62ea62 2024-12-03T06:14:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/cff7fa7040b9479bba6c6702130aa326 is 50, key is test_row_0/C:col10/1733206449995/Put/seqid=0 2024-12-03T06:14:11,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742116_1292 (size=12151) 2024-12-03T06:14:11,135 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/cff7fa7040b9479bba6c6702130aa326 2024-12-03T06:14:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bce2ede5b77d405882bc85464a2c339b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b 2024-12-03T06:14:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:11,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:11,146 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T06:14:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/8f376ac67b6a405bb7eb8c1e0e62ea62 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62 2024-12-03T06:14:11,151 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T06:14:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/cff7fa7040b9479bba6c6702130aa326 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326 2024-12-03T06:14:11,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206511154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206511155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206511157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206511157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,163 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T06:14:11,165 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5535461f8d6a19172fe21e7d7b6b77aa in 185ms, sequenceid=171, compaction requested=true 2024-12-03T06:14:11,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:11,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-03T06:14:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-03T06:14:11,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T06:14:11,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 493 msec 2024-12-03T06:14:11,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 500 msec 2024-12-03T06:14:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:11,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:11,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:11,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/47fd9329b0e74b13a2ee333d518c7f72 is 50, key is test_row_0/A:col10/1733206451154/Put/seqid=0 2024-12-03T06:14:11,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742117_1293 (size=12151) 2024-12-03T06:14:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T06:14:11,300 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:11,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-03T06:14:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:11,301 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:11,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:11,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206511310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206511310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206511311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206511311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,355 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/04e0e9b731174784b2991ae55d1439bc as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/04e0e9b731174784b2991ae55d1439bc 2024-12-03T06:14:11,355 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b37e3e5d8ef847879ebbe41f31a78d79 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b37e3e5d8ef847879ebbe41f31a78d79 2024-12-03T06:14:11,362 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into 04e0e9b731174784b2991ae55d1439bc(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:11,362 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=13, startTime=1733206450861; duration=0sec 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:11,362 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into b37e3e5d8ef847879ebbe41f31a78d79(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:11,362 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=13, startTime=1733206450860; duration=0sec 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:11,362 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:11,364 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:11,364 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:11,364 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,365 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f2991f24616e4cdaab300185fa2a4280, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=47.6 K 2024-12-03T06:14:11,365 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f2991f24616e4cdaab300185fa2a4280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206447596 2024-12-03T06:14:11,365 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9d8700e4784cf79ec2e93d129c9cd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206448656 2024-12-03T06:14:11,366 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1cf5ba797c4e6294209e4bafe37f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733206449325 2024-12-03T06:14:11,366 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cff7fa7040b9479bba6c6702130aa326, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733206449980 2024-12-03T06:14:11,392 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:11,392 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e06612314a3f4a9aace26d2e335ddb1e is 50, key is test_row_0/C:col10/1733206449995/Put/seqid=0 2024-12-03T06:14:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742118_1294 (size=12527) 2024-12-03T06:14:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:11,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206511419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206511420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206511420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206511420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:11,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:11,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206511630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206511630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206511630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206511630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/47fd9329b0e74b13a2ee333d518c7f72 2024-12-03T06:14:11,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/574a82a90b304be5bb56ba72a121eb2a is 50, key is test_row_0/B:col10/1733206451154/Put/seqid=0 2024-12-03T06:14:11,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742119_1295 (size=12151) 2024-12-03T06:14:11,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/574a82a90b304be5bb56ba72a121eb2a 2024-12-03T06:14:11,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c6623f8620984e38b8808f9c93737239 is 50, key is test_row_0/C:col10/1733206451154/Put/seqid=0 2024-12-03T06:14:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742120_1296 (size=12151) 2024-12-03T06:14:11,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:11,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206511779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,782 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 5535461f8d6a19172fe21e7d7b6b77aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,804 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e06612314a3f4a9aace26d2e335ddb1e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e06612314a3f4a9aace26d2e335ddb1e 2024-12-03T06:14:11,809 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into e06612314a3f4a9aace26d2e335ddb1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:11,809 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:11,809 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=12, startTime=1733206450861; duration=0sec 2024-12-03T06:14:11,809 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:11,809 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:11,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:11,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:11,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206511940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206511940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206511940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206511940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:12,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c6623f8620984e38b8808f9c93737239 2024-12-03T06:14:12,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/47fd9329b0e74b13a2ee333d518c7f72 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72 2024-12-03T06:14:12,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72, entries=150, sequenceid=189, filesize=11.9 K 2024-12-03T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/574a82a90b304be5bb56ba72a121eb2a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a 2024-12-03T06:14:12,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a, entries=150, sequenceid=189, filesize=11.9 K 2024-12-03T06:14:12,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c6623f8620984e38b8808f9c93737239 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239 2024-12-03T06:14:12,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239, entries=150, sequenceid=189, filesize=11.9 K 2024-12-03T06:14:12,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5535461f8d6a19172fe21e7d7b6b77aa in 877ms, sequenceid=189, compaction requested=true 2024-12-03T06:14:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:12,149 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:12,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:12,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:12,150 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:12,151 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:12,151 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:12,151 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,151 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b37e3e5d8ef847879ebbe41f31a78d79, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.9 K 2024-12-03T06:14:12,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:12,151 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:12,151 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:12,152 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,152 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/04e0e9b731174784b2991ae55d1439bc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=35.9 K 2024-12-03T06:14:12,152 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37e3e5d8ef847879ebbe41f31a78d79, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733206449325 2024-12-03T06:14:12,152 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e0e9b731174784b2991ae55d1439bc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733206449325 2024-12-03T06:14:12,152 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce2ede5b77d405882bc85464a2c339b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733206449980 2024-12-03T06:14:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:12,152 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47fd9329b0e74b13a2ee333d518c7f72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733206451154 2024-12-03T06:14:12,154 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f376ac67b6a405bb7eb8c1e0e62ea62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733206449980 2024-12-03T06:14:12,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:12,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:12,155 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 574a82a90b304be5bb56ba72a121eb2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733206451154 2024-12-03T06:14:12,171 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:12,172 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/9b5549ca6e114ca99221e7e03cbbb7d6 is 50, key is test_row_0/A:col10/1733206451154/Put/seqid=0 2024-12-03T06:14:12,172 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:12,173 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/5c1a6de412984ce0b2cd8206ee2debcb is 50, key is test_row_0/B:col10/1733206451154/Put/seqid=0 2024-12-03T06:14:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742121_1297 (size=12595) 2024-12-03T06:14:12,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742122_1298 (size=12595) 2024-12-03T06:14:12,193 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/9b5549ca6e114ca99221e7e03cbbb7d6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/9b5549ca6e114ca99221e7e03cbbb7d6 2024-12-03T06:14:12,200 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/5c1a6de412984ce0b2cd8206ee2debcb as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5c1a6de412984ce0b2cd8206ee2debcb 2024-12-03T06:14:12,201 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into 9b5549ca6e114ca99221e7e03cbbb7d6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:12,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:12,201 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=13, startTime=1733206452149; duration=0sec 2024-12-03T06:14:12,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:12,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:12,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T06:14:12,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T06:14:12,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T06:14:12,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. because compaction request was cancelled 2024-12-03T06:14:12,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:12,208 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into 5c1a6de412984ce0b2cd8206ee2debcb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:12,208 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:12,208 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=13, startTime=1733206452150; duration=0sec 2024-12-03T06:14:12,209 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:12,209 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:12,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,219 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:12,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bf1ae9514faf4638881e9007b0bc05de is 50, key is test_row_0/A:col10/1733206451289/Put/seqid=0 2024-12-03T06:14:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742123_1299 (size=12151) 2024-12-03T06:14:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:12,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:12,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206512466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206512488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206512488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206512488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206512599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206512599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206512599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206512599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,631 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bf1ae9514faf4638881e9007b0bc05de 2024-12-03T06:14:12,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/283c9f600e9c488588ca403f78807674 is 50, key is test_row_0/B:col10/1733206451289/Put/seqid=0 2024-12-03T06:14:12,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742124_1300 (size=12151) 2024-12-03T06:14:12,668 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/283c9f600e9c488588ca403f78807674 2024-12-03T06:14:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c2fb49b8f13c409fb90d526d196347ca is 50, key is test_row_0/C:col10/1733206451289/Put/seqid=0 2024-12-03T06:14:12,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742125_1301 (size=12151) 2024-12-03T06:14:12,696 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c2fb49b8f13c409fb90d526d196347ca 2024-12-03T06:14:12,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/bf1ae9514faf4638881e9007b0bc05de as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de 2024-12-03T06:14:12,707 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T06:14:12,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/283c9f600e9c488588ca403f78807674 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674 2024-12-03T06:14:12,714 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T06:14:12,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c2fb49b8f13c409fb90d526d196347ca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca 2024-12-03T06:14:12,722 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T06:14:12,724 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5535461f8d6a19172fe21e7d7b6b77aa in 505ms, sequenceid=209, compaction requested=true 2024-12-03T06:14:12,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:12,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:12,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-03T06:14:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-03T06:14:12,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T06:14:12,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-12-03T06:14:12,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4290 sec 2024-12-03T06:14:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:12,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206512830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206512831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/89bae42ba81d48978389dd6b7aa8611c is 50, key is test_row_0/A:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206512840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206512847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742126_1302 (size=12151) 2024-12-03T06:14:12,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/89bae42ba81d48978389dd6b7aa8611c 2024-12-03T06:14:12,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/fe559d05834d42868e3d23959064313e is 50, key is test_row_0/B:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206512950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206512950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206512950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742127_1303 (size=12151) 2024-12-03T06:14:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206512960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206513169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/fe559d05834d42868e3d23959064313e 2024-12-03T06:14:13,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/8aef348174ba4afd809af52557b50155 is 50, key is test_row_0/C:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742128_1304 (size=12151) 2024-12-03T06:14:13,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/8aef348174ba4afd809af52557b50155 2024-12-03T06:14:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/89bae42ba81d48978389dd6b7aa8611c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c 2024-12-03T06:14:13,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T06:14:13,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/fe559d05834d42868e3d23959064313e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e 2024-12-03T06:14:13,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T06:14:13,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/8aef348174ba4afd809af52557b50155 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155 2024-12-03T06:14:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T06:14:13,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:13,450 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-03T06:14:13,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T06:14:13,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5535461f8d6a19172fe21e7d7b6b77aa in 640ms, sequenceid=228, compaction requested=true 2024-12-03T06:14:13,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:13,452 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:13,452 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:13,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:13,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:13,453 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:13,453 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:13,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:13,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:13,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:13,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:13,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:13,454 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:13,455 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:13,455 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,455 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/9b5549ca6e114ca99221e7e03cbbb7d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=36.0 K 2024-12-03T06:14:13,456 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:13,456 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:13,456 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,456 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5c1a6de412984ce0b2cd8206ee2debcb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=36.0 K 2024-12-03T06:14:13,456 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5549ca6e114ca99221e7e03cbbb7d6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733206451154 2024-12-03T06:14:13,456 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1a6de412984ce0b2cd8206ee2debcb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733206451154 2024-12-03T06:14:13,457 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1ae9514faf4638881e9007b0bc05de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733206451289 2024-12-03T06:14:13,457 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 283c9f600e9c488588ca403f78807674, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733206451289 2024-12-03T06:14:13,458 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting fe559d05834d42868e3d23959064313e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:13,458 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89bae42ba81d48978389dd6b7aa8611c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:13,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:14:13,473 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#258 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:13,474 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/aaf0f9c0402a45b8aa1a0b24f3893d95 is 50, key is test_row_0/B:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:13,476 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:13,476 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/7e18792a95f247dfa3f06f4996947426 is 50, key is test_row_0/A:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:13,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b58cdb41964e4380847f2e1714590d53 is 50, key is test_row_0/A:col10/1733206452832/Put/seqid=0 2024-12-03T06:14:13,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742129_1305 (size=12697) 2024-12-03T06:14:13,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742131_1307 (size=12151) 2024-12-03T06:14:13,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742130_1306 (size=12697) 2024-12-03T06:14:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206513492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206513494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206513494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206513498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206513599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206513599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206513599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206513610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206513810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206513810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206513811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206513819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:13,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b58cdb41964e4380847f2e1714590d53 2024-12-03T06:14:13,898 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/aaf0f9c0402a45b8aa1a0b24f3893d95 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/aaf0f9c0402a45b8aa1a0b24f3893d95 2024-12-03T06:14:13,899 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/7e18792a95f247dfa3f06f4996947426 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/7e18792a95f247dfa3f06f4996947426 2024-12-03T06:14:13,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/52c85d2e49844cb6bc0de5cd396fb515 is 50, key is test_row_0/B:col10/1733206452832/Put/seqid=0 2024-12-03T06:14:13,908 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into aaf0f9c0402a45b8aa1a0b24f3893d95(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:13,908 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:13,908 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=13, startTime=1733206453452; duration=0sec 2024-12-03T06:14:13,908 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:13,908 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:13,908 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:13,910 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into 7e18792a95f247dfa3f06f4996947426(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:13,910 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:13,910 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=13, startTime=1733206453452; duration=0sec 2024-12-03T06:14:13,910 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:13,910 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:13,910 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:13,910 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:13,910 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,910 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e06612314a3f4a9aace26d2e335ddb1e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=47.8 K 2024-12-03T06:14:13,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742132_1308 (size=12151) 2024-12-03T06:14:13,911 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e06612314a3f4a9aace26d2e335ddb1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733206449980 2024-12-03T06:14:13,911 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c6623f8620984e38b8808f9c93737239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733206451154 2024-12-03T06:14:13,911 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c2fb49b8f13c409fb90d526d196347ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733206451289 2024-12-03T06:14:13,911 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aef348174ba4afd809af52557b50155, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:13,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:13,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:13,929 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:13,929 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b36ca060afaa4eb4b844ea60ad0ad631 is 50, key is test_row_0/C:col10/1733206452463/Put/seqid=0 2024-12-03T06:14:13,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742133_1309 (size=12663) 2024-12-03T06:14:13,956 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b36ca060afaa4eb4b844ea60ad0ad631 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b36ca060afaa4eb4b844ea60ad0ad631 2024-12-03T06:14:13,961 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into b36ca060afaa4eb4b844ea60ad0ad631(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:13,961 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:13,961 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=12, startTime=1733206453453; duration=0sec 2024-12-03T06:14:13,961 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:13,961 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:14,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206514120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206514120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206514120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206514129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:14,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/52c85d2e49844cb6bc0de5cd396fb515 2024-12-03T06:14:14,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/fdbf2cd1b35a46debe35a41fce133bbd is 50, key is test_row_0/C:col10/1733206452832/Put/seqid=0 2024-12-03T06:14:14,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742134_1310 (size=12151) 2024-12-03T06:14:14,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/fdbf2cd1b35a46debe35a41fce133bbd 2024-12-03T06:14:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b58cdb41964e4380847f2e1714590d53 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53 2024-12-03T06:14:14,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T06:14:14,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/52c85d2e49844cb6bc0de5cd396fb515 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515 2024-12-03T06:14:14,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T06:14:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/fdbf2cd1b35a46debe35a41fce133bbd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd 2024-12-03T06:14:14,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T06:14:14,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5535461f8d6a19172fe21e7d7b6b77aa in 864ms, sequenceid=244, compaction requested=false 2024-12-03T06:14:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:14,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,379 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/aea44c8baba443558aca73d7ecfe3b23 is 50, key is test_row_0/A:col10/1733206453493/Put/seqid=0 2024-12-03T06:14:14,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742135_1311 (size=12251) 2024-12-03T06:14:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206514642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206514643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206514643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206514646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206514749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206514749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206514749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206514759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,787 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/aea44c8baba443558aca73d7ecfe3b23 2024-12-03T06:14:14,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/7e373577a64648b4ad595d29d656bf72 is 50, key is test_row_0/B:col10/1733206453493/Put/seqid=0 2024-12-03T06:14:14,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742136_1312 (size=12251) 2024-12-03T06:14:14,800 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/7e373577a64648b4ad595d29d656bf72 2024-12-03T06:14:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/32b98ae3d50b4078834164ffbf60721a is 50, key is test_row_0/C:col10/1733206453493/Put/seqid=0 2024-12-03T06:14:14,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742137_1313 (size=12251) 2024-12-03T06:14:14,812 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/32b98ae3d50b4078834164ffbf60721a 2024-12-03T06:14:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/aea44c8baba443558aca73d7ecfe3b23 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23 2024-12-03T06:14:14,820 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T06:14:14,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/7e373577a64648b4ad595d29d656bf72 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72 2024-12-03T06:14:14,832 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T06:14:14,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/32b98ae3d50b4078834164ffbf60721a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a 2024-12-03T06:14:14,839 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T06:14:14,841 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5535461f8d6a19172fe21e7d7b6b77aa in 461ms, sequenceid=263, compaction requested=true 2024-12-03T06:14:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-03T06:14:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-03T06:14:14,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T06:14:14,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-12-03T06:14:14,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3960 sec 2024-12-03T06:14:14,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:14:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:14,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:14,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/80258f8f7e4d46c397d1d4f31537aab8 is 50, key is test_row_0/A:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742138_1314 (size=12301) 2024-12-03T06:14:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206514989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206514991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206514993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206514993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206515100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206515110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206515110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206515110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206515309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206515319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206515319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206515319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/80258f8f7e4d46c397d1d4f31537aab8 2024-12-03T06:14:15,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1f0c546d62f2465da0ba4904c9cb338d is 50, key is test_row_0/B:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:15,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742139_1315 (size=12301) 2024-12-03T06:14:15,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1f0c546d62f2465da0ba4904c9cb338d 2024-12-03T06:14:15,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea68c28f59f54270aea6aaeba1b78a1a is 50, key is test_row_0/C:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742140_1316 (size=12301) 2024-12-03T06:14:15,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea68c28f59f54270aea6aaeba1b78a1a 2024-12-03T06:14:15,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/80258f8f7e4d46c397d1d4f31537aab8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8 2024-12-03T06:14:15,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:14:15,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1f0c546d62f2465da0ba4904c9cb338d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d 2024-12-03T06:14:15,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:14:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea68c28f59f54270aea6aaeba1b78a1a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a 2024-12-03T06:14:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T06:14:15,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5535461f8d6a19172fe21e7d7b6b77aa in 516ms, sequenceid=281, compaction requested=true 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:15,479 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:15,479 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:15,480 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:15,480 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:15,480 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,480 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/7e18792a95f247dfa3f06f4996947426, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=48.2 K 2024-12-03T06:14:15,481 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e18792a95f247dfa3f06f4996947426, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:15,481 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:15,481 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:15,481 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,482 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/aaf0f9c0402a45b8aa1a0b24f3893d95, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=48.2 K 2024-12-03T06:14:15,482 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b58cdb41964e4380847f2e1714590d53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733206452832 2024-12-03T06:14:15,482 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf0f9c0402a45b8aa1a0b24f3893d95, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:15,482 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea44c8baba443558aca73d7ecfe3b23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206453491 2024-12-03T06:14:15,482 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c85d2e49844cb6bc0de5cd396fb515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733206452832 2024-12-03T06:14:15,483 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e373577a64648b4ad595d29d656bf72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206453491 2024-12-03T06:14:15,483 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80258f8f7e4d46c397d1d4f31537aab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:15,484 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0c546d62f2465da0ba4904c9cb338d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:15,494 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:15,495 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/a46ce579e7fb48689e6674eb09ad692c is 50, key is test_row_0/B:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:15,496 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:15,496 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/6326a11b11bd4761b0f4b273d3c369b7 is 50, key is test_row_0/A:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:15,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742141_1317 (size=12983) 2024-12-03T06:14:15,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742142_1318 (size=12983) 2024-12-03T06:14:15,524 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/6326a11b11bd4761b0f4b273d3c369b7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6326a11b11bd4761b0f4b273d3c369b7 2024-12-03T06:14:15,530 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into 6326a11b11bd4761b0f4b273d3c369b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:15,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:15,530 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=12, startTime=1733206455479; duration=0sec 2024-12-03T06:14:15,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:15,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:15,530 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:15,532 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:15,532 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:15,532 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,532 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b36ca060afaa4eb4b844ea60ad0ad631, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=48.2 K 2024-12-03T06:14:15,533 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36ca060afaa4eb4b844ea60ad0ad631, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733206452463 2024-12-03T06:14:15,533 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdbf2cd1b35a46debe35a41fce133bbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733206452832 2024-12-03T06:14:15,533 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b98ae3d50b4078834164ffbf60721a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733206453491 2024-12-03T06:14:15,533 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea68c28f59f54270aea6aaeba1b78a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:15,543 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:15,544 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/524fe359962a4eae825158a17e863de9 is 50, key is test_row_0/C:col10/1733206454642/Put/seqid=0 2024-12-03T06:14:15,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742143_1319 (size=12949) 2024-12-03T06:14:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T06:14:15,600 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:15,601 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/524fe359962a4eae825158a17e863de9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/524fe359962a4eae825158a17e863de9 2024-12-03T06:14:15,601 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-03T06:14:15,603 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:15,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:15,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:15,619 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into 524fe359962a4eae825158a17e863de9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:15,619 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:15,619 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=12, startTime=1733206455479; duration=0sec 2024-12-03T06:14:15,621 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:15,621 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:15,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:15,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:15,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/6a49c0470d9148f592e2ae58237de4a7 is 50, key is test_row_0/A:col10/1733206454991/Put/seqid=0 2024-12-03T06:14:15,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742144_1320 (size=14741) 2024-12-03T06:14:15,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/6a49c0470d9148f592e2ae58237de4a7 2024-12-03T06:14:15,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206515653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206515655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206515655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206515657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/693a24343f5e42c590845b35e57d2e39 is 50, key is test_row_0/B:col10/1733206454991/Put/seqid=0 2024-12-03T06:14:15,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742145_1321 (size=12301) 2024-12-03T06:14:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:15,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206515770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206515770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206515770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206515770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1733206515820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,822 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in 5535461f8d6a19172fe21e7d7b6b77aa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:15,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:15,916 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/a46ce579e7fb48689e6674eb09ad692c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a46ce579e7fb48689e6674eb09ad692c 2024-12-03T06:14:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:15,920 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into a46ce579e7fb48689e6674eb09ad692c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:15,920 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:15,920 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=12, startTime=1733206455479; duration=0sec 2024-12-03T06:14:15,921 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:15,921 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206515979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206515979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206515979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206515979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/693a24343f5e42c590845b35e57d2e39 2024-12-03T06:14:16,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e45812c4340141f1a40a338417f99b82 is 50, key is test_row_0/C:col10/1733206454991/Put/seqid=0 2024-12-03T06:14:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742146_1322 (size=12301) 2024-12-03T06:14:16,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206516290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206516290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206516290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206516290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:16,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e45812c4340141f1a40a338417f99b82 2024-12-03T06:14:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/6a49c0470d9148f592e2ae58237de4a7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7 2024-12-03T06:14:16,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7, entries=200, sequenceid=299, filesize=14.4 K 2024-12-03T06:14:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/693a24343f5e42c590845b35e57d2e39 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39 2024-12-03T06:14:16,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T06:14:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e45812c4340141f1a40a338417f99b82 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82 2024-12-03T06:14:16,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T06:14:16,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5535461f8d6a19172fe21e7d7b6b77aa in 940ms, sequenceid=299, compaction requested=false 2024-12-03T06:14:16,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:16,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T06:14:16,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:16,680 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:16,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/205c493caf5c48d6b71aceef4eeef484 is 50, key is test_row_0/A:col10/1733206455655/Put/seqid=0 2024-12-03T06:14:16,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742147_1323 (size=12301) 2024-12-03T06:14:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:16,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206516816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206516817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206516817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206516818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206516929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206516929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206516929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206516929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,089 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/205c493caf5c48d6b71aceef4eeef484 2024-12-03T06:14:17,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/5fd789f3476f4ffba97702a02b2697d9 is 50, key is test_row_0/B:col10/1733206455655/Put/seqid=0 2024-12-03T06:14:17,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742148_1324 (size=12301) 2024-12-03T06:14:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206517139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206517140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206517140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206517140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206517450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206517450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206517450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206517451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,505 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/5fd789f3476f4ffba97702a02b2697d9 2024-12-03T06:14:17,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea895abd03164656aa3fb0fcd24826ca is 50, key is test_row_0/C:col10/1733206455655/Put/seqid=0 2024-12-03T06:14:17,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742149_1325 (size=12301) 2024-12-03T06:14:17,567 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea895abd03164656aa3fb0fcd24826ca 2024-12-03T06:14:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/205c493caf5c48d6b71aceef4eeef484 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484 2024-12-03T06:14:17,579 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T06:14:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/5fd789f3476f4ffba97702a02b2697d9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9 2024-12-03T06:14:17,589 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T06:14:17,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/ea895abd03164656aa3fb0fcd24826ca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca 2024-12-03T06:14:17,599 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T06:14:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5535461f8d6a19172fe21e7d7b6b77aa in 920ms, sequenceid=316, compaction requested=true 2024-12-03T06:14:17,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:17,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:17,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-03T06:14:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-03T06:14:17,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-03T06:14:17,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9970 sec 2024-12-03T06:14:17,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0050 sec 2024-12-03T06:14:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T06:14:17,751 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:17,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-12-03T06:14:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T06:14:17,758 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:17,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:17,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T06:14:17,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-03T06:14:17,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:17,912 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:17,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/fdabecba39ae4bfe929135588f468d99 is 50, key is test_row_0/A:col10/1733206456814/Put/seqid=0 2024-12-03T06:14:17,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742150_1326 (size=12301) 2024-12-03T06:14:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:17,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206517974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206517974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206517974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206517975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T06:14:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206518090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206518090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206518090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206518090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206518299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206518299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206518300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206518300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,321 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/fdabecba39ae4bfe929135588f468d99 2024-12-03T06:14:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4ce830e8005f460481ee8635f5e2205d is 50, key is test_row_0/B:col10/1733206456814/Put/seqid=0 2024-12-03T06:14:18,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742151_1327 (size=12301) 2024-12-03T06:14:18,337 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4ce830e8005f460481ee8635f5e2205d 2024-12-03T06:14:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/2d5161ff408541a582218e819d949721 is 50, key is test_row_0/C:col10/1733206456814/Put/seqid=0 2024-12-03T06:14:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742152_1328 (size=12301) 2024-12-03T06:14:18,369 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/2d5161ff408541a582218e819d949721 2024-12-03T06:14:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/fdabecba39ae4bfe929135588f468d99 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99 2024-12-03T06:14:18,378 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99, entries=150, sequenceid=332, filesize=12.0 K 2024-12-03T06:14:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T06:14:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/4ce830e8005f460481ee8635f5e2205d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d 2024-12-03T06:14:18,386 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d, entries=150, sequenceid=332, filesize=12.0 K 2024-12-03T06:14:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/2d5161ff408541a582218e819d949721 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721 2024-12-03T06:14:18,393 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721, entries=150, sequenceid=332, filesize=12.0 K 2024-12-03T06:14:18,394 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5535461f8d6a19172fe21e7d7b6b77aa in 483ms, sequenceid=332, compaction requested=true 2024-12-03T06:14:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-12-03T06:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-12-03T06:14:18,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T06:14:18,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 637 msec 2024-12-03T06:14:18,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 646 msec 2024-12-03T06:14:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:18,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e3a9608b7f8d45199883cdd9b8a26240 is 50, key is test_row_0/A:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742153_1329 (size=14741) 2024-12-03T06:14:18,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e3a9608b7f8d45199883cdd9b8a26240 2024-12-03T06:14:18,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9bc443c3ca4047da96f404ef3a82d804 is 50, key is test_row_0/B:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742154_1330 (size=12301) 2024-12-03T06:14:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206518626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206518628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206518627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9bc443c3ca4047da96f404ef3a82d804 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206518631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/69486bf1d449442fad7ab9782b5ed782 is 50, key is test_row_0/C:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:18,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742155_1331 (size=12301) 2024-12-03T06:14:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206518740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206518740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206518740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206518750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T06:14:18,890 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:18,891 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-12-03T06:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:18,892 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:18,893 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:18,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206518949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206518949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206518949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:18,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206518959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:19,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-12-03T06:14:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:19,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/69486bf1d449442fad7ab9782b5ed782 2024-12-03T06:14:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/e3a9608b7f8d45199883cdd9b8a26240 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240 2024-12-03T06:14:19,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240, entries=200, sequenceid=351, filesize=14.4 K 2024-12-03T06:14:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/9bc443c3ca4047da96f404ef3a82d804 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804 2024-12-03T06:14:19,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T06:14:19,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/69486bf1d449442fad7ab9782b5ed782 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782 2024-12-03T06:14:19,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T06:14:19,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5535461f8d6a19172fe21e7d7b6b77aa in 488ms, sequenceid=351, compaction requested=true 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:19,101 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:19,101 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:19,102 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:14:19,102 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67067 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:14:19,102 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:19,102 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:19,102 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,102 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,102 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6326a11b11bd4761b0f4b273d3c369b7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=65.5 K 2024-12-03T06:14:19,102 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a46ce579e7fb48689e6674eb09ad692c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=60.7 K 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6326a11b11bd4761b0f4b273d3c369b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a46ce579e7fb48689e6674eb09ad692c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a49c0470d9148f592e2ae58237de4a7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733206454989 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 693a24343f5e42c590845b35e57d2e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733206454989 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 205c493caf5c48d6b71aceef4eeef484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733206455652 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd789f3476f4ffba97702a02b2697d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733206455652 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdabecba39ae4bfe929135588f468d99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733206456814 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce830e8005f460481ee8635f5e2205d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733206456814 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a9608b7f8d45199883cdd9b8a26240, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:19,103 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bc443c3ca4047da96f404ef3a82d804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:19,111 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:19,111 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:19,112 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/91a4cca0509b41f98eca641d5cb9839b is 50, key is test_row_0/A:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:19,112 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/98eec2feebd14e47816022819ead50e6 is 50, key is test_row_0/B:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742157_1333 (size=13153) 2024-12-03T06:14:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742156_1332 (size=13153) 2024-12-03T06:14:19,128 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/91a4cca0509b41f98eca641d5cb9839b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/91a4cca0509b41f98eca641d5cb9839b 2024-12-03T06:14:19,129 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/98eec2feebd14e47816022819ead50e6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/98eec2feebd14e47816022819ead50e6 2024-12-03T06:14:19,133 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into 91a4cca0509b41f98eca641d5cb9839b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:19,133 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:19,133 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=11, startTime=1733206459101; duration=0sec 2024-12-03T06:14:19,133 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:19,133 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:19,133 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:14:19,134 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into 98eec2feebd14e47816022819ead50e6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:19,134 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:19,134 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=11, startTime=1733206459101; duration=0sec 2024-12-03T06:14:19,134 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:19,134 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:19,136 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:14:19,136 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:19,136 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,136 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/524fe359962a4eae825158a17e863de9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=60.7 K 2024-12-03T06:14:19,136 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524fe359962a4eae825158a17e863de9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733206454642 2024-12-03T06:14:19,137 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45812c4340141f1a40a338417f99b82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733206454989 2024-12-03T06:14:19,137 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea895abd03164656aa3fb0fcd24826ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733206455652 2024-12-03T06:14:19,138 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5161ff408541a582218e819d949721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733206456814 2024-12-03T06:14:19,138 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69486bf1d449442fad7ab9782b5ed782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:19,146 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#287 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:19,147 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/93ee0fe0af6b4402886213edebb8bbec is 50, key is test_row_0/C:col10/1733206457974/Put/seqid=0 2024-12-03T06:14:19,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742158_1334 (size=13119) 2024-12-03T06:14:19,163 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/93ee0fe0af6b4402886213edebb8bbec as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/93ee0fe0af6b4402886213edebb8bbec 2024-12-03T06:14:19,168 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into 93ee0fe0af6b4402886213edebb8bbec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:19,168 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:19,168 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=11, startTime=1733206459101; duration=0sec 2024-12-03T06:14:19,168 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:19,168 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:19,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-12-03T06:14:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:19,199 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b67f235a7c5342b59e41aedbb993c6ab is 50, key is test_row_1/A:col10/1733206458625/Put/seqid=0 2024-12-03T06:14:19,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742159_1335 (size=9857) 2024-12-03T06:14:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:19,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206519276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206519279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206519280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206519280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206519390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206519390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206519390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206519390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206519600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206519600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206519602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206519602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,607 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b67f235a7c5342b59e41aedbb993c6ab 2024-12-03T06:14:19,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/dea710ff9d9c4001a32c963ab5c8d7e7 is 50, key is test_row_1/B:col10/1733206458625/Put/seqid=0 2024-12-03T06:14:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742160_1336 (size=9857) 2024-12-03T06:14:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:20,060 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/dea710ff9d9c4001a32c963ab5c8d7e7 2024-12-03T06:14:20,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e5bdcff2f64447e89ae6788d3282472e is 50, key is test_row_1/C:col10/1733206458625/Put/seqid=0 2024-12-03T06:14:20,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742161_1337 (size=9857) 2024-12-03T06:14:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206520419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206520419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206520419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206520419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:20,486 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e5bdcff2f64447e89ae6788d3282472e 2024-12-03T06:14:20,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/b67f235a7c5342b59e41aedbb993c6ab as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab 2024-12-03T06:14:20,496 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab, entries=100, sequenceid=367, filesize=9.6 K 2024-12-03T06:14:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/dea710ff9d9c4001a32c963ab5c8d7e7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7 2024-12-03T06:14:20,503 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7, entries=100, sequenceid=367, filesize=9.6 K 2024-12-03T06:14:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/e5bdcff2f64447e89ae6788d3282472e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e 2024-12-03T06:14:20,510 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e, entries=100, sequenceid=367, filesize=9.6 K 2024-12-03T06:14:20,511 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5535461f8d6a19172fe21e7d7b6b77aa in 1313ms, sequenceid=367, compaction requested=false 2024-12-03T06:14:20,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:20,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:20,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-12-03T06:14:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-12-03T06:14:20,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-03T06:14:20,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-12-03T06:14:20,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.6280 sec 2024-12-03T06:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T06:14:21,040 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:21,040 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-12-03T06:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T06:14:21,042 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:21,043 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:21,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T06:14:21,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:21,195 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/20d043e73e334e8ba3009a3cea7a3a95 is 50, key is test_row_0/A:col10/1733206459279/Put/seqid=0 2024-12-03T06:14:21,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742162_1338 (size=12301) 2024-12-03T06:14:21,204 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/20d043e73e334e8ba3009a3cea7a3a95 2024-12-03T06:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/b5e89d1b3f48447b85e4e5e7afcbdb01 is 50, key is test_row_0/B:col10/1733206459279/Put/seqid=0 2024-12-03T06:14:21,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742163_1339 (size=12301) 2024-12-03T06:14:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T06:14:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:21,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206521437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206521438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206521439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206521440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206521549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206521549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206521549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206521549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,626 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/b5e89d1b3f48447b85e4e5e7afcbdb01 2024-12-03T06:14:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/f413e786ba9f4362b85884c0d1591070 is 50, key is test_row_0/C:col10/1733206459279/Put/seqid=0 2024-12-03T06:14:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742164_1340 (size=12301) 2024-12-03T06:14:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T06:14:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206521759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206521759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206521759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206521759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,036 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/f413e786ba9f4362b85884c0d1591070 2024-12-03T06:14:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/20d043e73e334e8ba3009a3cea7a3a95 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95 2024-12-03T06:14:22,046 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95, entries=150, sequenceid=387, filesize=12.0 K 2024-12-03T06:14:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/b5e89d1b3f48447b85e4e5e7afcbdb01 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01 2024-12-03T06:14:22,053 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01, entries=150, sequenceid=387, filesize=12.0 K 2024-12-03T06:14:22,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/f413e786ba9f4362b85884c0d1591070 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070 2024-12-03T06:14:22,057 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070, entries=150, sequenceid=387, filesize=12.0 K 2024-12-03T06:14:22,058 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5535461f8d6a19172fe21e7d7b6b77aa in 863ms, sequenceid=387, compaction requested=true 2024-12-03T06:14:22,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:22,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-12-03T06:14:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-12-03T06:14:22,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-03T06:14:22,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-12-03T06:14:22,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.0220 sec 2024-12-03T06:14:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:22,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:22,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/5ad3ce81e286465cb6f4e2a2829d02a3 is 50, key is test_row_0/A:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:22,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742165_1341 (size=12301) 2024-12-03T06:14:22,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206522088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206522092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206522093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206522094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T06:14:22,179 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:22,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-12-03T06:14:22,182 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:22,182 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:22,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206522199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206522199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206522210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206522210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:22,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:22,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:22,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206522410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206522410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206522420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206522420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:22,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/5ad3ce81e286465cb6f4e2a2829d02a3 2024-12-03T06:14:22,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:22,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1800d42fd68e4d1a9bac73a0063cea12 is 50, key is test_row_0/B:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:22,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742166_1342 (size=12301) 2024-12-03T06:14:22,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206522720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206522720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206522730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206522730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:22,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1800d42fd68e4d1a9bac73a0063cea12 2024-12-03T06:14:22,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/312a309461be4d68b02ea232482609c9 is 50, key is test_row_0/C:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:22,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742167_1343 (size=12301) 2024-12-03T06:14:22,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:22,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:22,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,083 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-12-03T06:14:23,084 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:23,084 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,084 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:23,085 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-12-03T06:14:23,085 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:23,085 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,085 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:23,085 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-12-03T06:14:23,085 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:23,085 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,085 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:23,087 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-12-03T06:14:23,087 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:23,087 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,087 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:23,088 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-12-03T06:14:23,088 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:23,088 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,088 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:23,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:23,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58014 deadline: 1733206523229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1733206523230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:23,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:23,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1733206523240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57978 deadline: 1733206523240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:14:23,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:23,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/312a309461be4d68b02ea232482609c9 2024-12-03T06:14:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/5ad3ce81e286465cb6f4e2a2829d02a3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3 2024-12-03T06:14:23,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T06:14:23,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/1800d42fd68e4d1a9bac73a0063cea12 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12 2024-12-03T06:14:23,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T06:14:23,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/312a309461be4d68b02ea232482609c9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9 2024-12-03T06:14:23,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T06:14:23,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5535461f8d6a19172fe21e7d7b6b77aa in 1297ms, sequenceid=401, compaction requested=true 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:23,369 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5535461f8d6a19172fe21e7d7b6b77aa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:23,369 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:23,370 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:23,370 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/B is initiating minor compaction (all files) 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/A is initiating minor compaction (all files) 2024-12-03T06:14:23,371 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/A in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,371 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/B in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,371 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/91a4cca0509b41f98eca641d5cb9839b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=46.5 K 2024-12-03T06:14:23,371 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/98eec2feebd14e47816022819ead50e6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=46.5 K 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a4cca0509b41f98eca641d5cb9839b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 98eec2feebd14e47816022819ead50e6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67f235a7c5342b59e41aedbb993c6ab, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733206458625 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting dea710ff9d9c4001a32c963ab5c8d7e7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733206458625 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d043e73e334e8ba3009a3cea7a3a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733206459273 2024-12-03T06:14:23,371 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e89d1b3f48447b85e4e5e7afcbdb01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733206459273 2024-12-03T06:14:23,372 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad3ce81e286465cb6f4e2a2829d02a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733206461437 2024-12-03T06:14:23,372 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1800d42fd68e4d1a9bac73a0063cea12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733206461437 2024-12-03T06:14:23,378 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:23,379 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/a52f2f0c8f0d41ac9390c7a10cc4a5b6 is 50, key is test_row_0/A:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:23,379 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:23,379 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/c6ddffe44e98450a9c3fc3ddfe48b370 is 50, key is test_row_0/B:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:23,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742168_1344 (size=13289) 2024-12-03T06:14:23,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742169_1345 (size=13289) 2024-12-03T06:14:23,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-12-03T06:14:23,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,452 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:23,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/f1cdd41c9f1c4397ae6795b841a5ea69 is 50, key is test_row_0/A:col10/1733206462082/Put/seqid=0 2024-12-03T06:14:23,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742170_1346 (size=12301) 2024-12-03T06:14:23,787 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/a52f2f0c8f0d41ac9390c7a10cc4a5b6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a52f2f0c8f0d41ac9390c7a10cc4a5b6 2024-12-03T06:14:23,787 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/c6ddffe44e98450a9c3fc3ddfe48b370 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c6ddffe44e98450a9c3fc3ddfe48b370 2024-12-03T06:14:23,792 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/A of 5535461f8d6a19172fe21e7d7b6b77aa into a52f2f0c8f0d41ac9390c7a10cc4a5b6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:23,792 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/B of 5535461f8d6a19172fe21e7d7b6b77aa into c6ddffe44e98450a9c3fc3ddfe48b370(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:23,792 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/A, priority=12, startTime=1733206463369; duration=0sec 2024-12-03T06:14:23,792 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/B, priority=12, startTime=1733206463369; duration=0sec 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:A 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:B 2024-12-03T06:14:23,792 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:23,793 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:23,794 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 5535461f8d6a19172fe21e7d7b6b77aa/C is initiating minor compaction (all files) 2024-12-03T06:14:23,794 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5535461f8d6a19172fe21e7d7b6b77aa/C in TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:23,794 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/93ee0fe0af6b4402886213edebb8bbec, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp, totalSize=46.5 K 2024-12-03T06:14:23,794 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ee0fe0af6b4402886213edebb8bbec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733206457963 2024-12-03T06:14:23,795 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5bdcff2f64447e89ae6788d3282472e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733206458625 2024-12-03T06:14:23,795 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f413e786ba9f4362b85884c0d1591070, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733206459273 2024-12-03T06:14:23,795 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 312a309461be4d68b02ea232482609c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733206461437 2024-12-03T06:14:23,803 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5535461f8d6a19172fe21e7d7b6b77aa#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:23,804 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b3131726ee8f47449c97624f92a7aaf0 is 50, key is test_row_0/C:col10/1733206462071/Put/seqid=0 2024-12-03T06:14:23,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742171_1347 (size=13255) 2024-12-03T06:14:23,868 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/f1cdd41c9f1c4397ae6795b841a5ea69 2024-12-03T06:14:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/f866bb731a7548f1ae4dfaf808918bf3 is 50, key is test_row_0/B:col10/1733206462082/Put/seqid=0 2024-12-03T06:14:23,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742172_1348 (size=12301) 2024-12-03T06:14:24,220 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/b3131726ee8f47449c97624f92a7aaf0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b3131726ee8f47449c97624f92a7aaf0 2024-12-03T06:14:24,226 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5535461f8d6a19172fe21e7d7b6b77aa/C of 5535461f8d6a19172fe21e7d7b6b77aa into b3131726ee8f47449c97624f92a7aaf0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:24,226 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:24,226 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., storeName=5535461f8d6a19172fe21e7d7b6b77aa/C, priority=12, startTime=1733206463369; duration=0sec 2024-12-03T06:14:24,227 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:24,227 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5535461f8d6a19172fe21e7d7b6b77aa:C 2024-12-03T06:14:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. as already flushing 2024-12-03T06:14:24,241 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-12-03T06:14:24,241 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:24,241 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,242 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:24,251 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-12-03T06:14:24,251 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:24,251 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-12-03T06:14:24,251 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-12-03T06:14:24,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:24,251 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:24,251 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:24,251 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:24,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:24,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:24,283 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/f866bb731a7548f1ae4dfaf808918bf3 2024-12-03T06:14:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c98a53880f4241ce81b57fdfae8326a2 is 50, key is test_row_0/C:col10/1733206462082/Put/seqid=0 2024-12-03T06:14:24,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742173_1349 (size=12301) 2024-12-03T06:14:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:24,700 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c98a53880f4241ce81b57fdfae8326a2 2024-12-03T06:14:24,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/f1cdd41c9f1c4397ae6795b841a5ea69 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/f1cdd41c9f1c4397ae6795b841a5ea69 2024-12-03T06:14:24,714 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/f1cdd41c9f1c4397ae6795b841a5ea69, entries=150, sequenceid=419, filesize=12.0 K 2024-12-03T06:14:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/f866bb731a7548f1ae4dfaf808918bf3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/f866bb731a7548f1ae4dfaf808918bf3 2024-12-03T06:14:24,720 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/f866bb731a7548f1ae4dfaf808918bf3, entries=150, sequenceid=419, filesize=12.0 K 2024-12-03T06:14:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/c98a53880f4241ce81b57fdfae8326a2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c98a53880f4241ce81b57fdfae8326a2 2024-12-03T06:14:24,725 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c98a53880f4241ce81b57fdfae8326a2, entries=150, sequenceid=419, filesize=12.0 K 2024-12-03T06:14:24,727 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 5535461f8d6a19172fe21e7d7b6b77aa in 1275ms, sequenceid=419, compaction requested=false 2024-12-03T06:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 5535461f8d6a19172fe21e7d7b6b77aa: 2024-12-03T06:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-12-03T06:14:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-12-03T06:14:24,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-03T06:14:24,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5470 sec 2024-12-03T06:14:24,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 2.5520 sec 2024-12-03T06:14:25,931 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-12-03T06:14:25,931 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:25,932 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:25,932 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:25,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T06:14:26,350 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3818 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3728 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3771 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3653 2024-12-03T06:14:26,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:14:26,350 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:14:26,351 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:14:26,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:26,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:26,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:26,352 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:14:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:26,356 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206466355"}]},"ts":"1733206466355"} 2024-12-03T06:14:26,357 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:14:26,357 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:14:26,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:14:26,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, UNASSIGN}] 2024-12-03T06:14:26,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, UNASSIGN 2024-12-03T06:14:26,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=5535461f8d6a19172fe21e7d7b6b77aa, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:26,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, UNASSIGN because future has completed 2024-12-03T06:14:26,364 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:14:26,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:14:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:26,517 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:26,517 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:14:26,517 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 5535461f8d6a19172fe21e7d7b6b77aa, disabling compactions & flushes 2024-12-03T06:14:26,517 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. after waiting 0 ms 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:26,518 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing 5535461f8d6a19172fe21e7d7b6b77aa 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=A 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=B 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5535461f8d6a19172fe21e7d7b6b77aa, store=C 2024-12-03T06:14:26,518 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:26,523 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/13c8e714077740418f44952d681ebe11 is 50, key is test_row_0/A:col10/1733206464240/Put/seqid=0 2024-12-03T06:14:26,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742174_1350 (size=12301) 2024-12-03T06:14:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:26,927 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/13c8e714077740418f44952d681ebe11 2024-12-03T06:14:26,934 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/234313d5f0e54b8296547a228e0b1e5d is 50, key is test_row_0/B:col10/1733206464240/Put/seqid=0 2024-12-03T06:14:26,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742175_1351 (size=12301) 2024-12-03T06:14:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:27,339 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/234313d5f0e54b8296547a228e0b1e5d 2024-12-03T06:14:27,345 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/77381b65495449f18fe54d9e7557a27e is 50, key is test_row_0/C:col10/1733206464240/Put/seqid=0 2024-12-03T06:14:27,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742176_1352 (size=12301) 2024-12-03T06:14:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:27,749 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/77381b65495449f18fe54d9e7557a27e 2024-12-03T06:14:27,788 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/A/13c8e714077740418f44952d681ebe11 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/13c8e714077740418f44952d681ebe11 2024-12-03T06:14:27,791 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/13c8e714077740418f44952d681ebe11, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T06:14:27,792 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/B/234313d5f0e54b8296547a228e0b1e5d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/234313d5f0e54b8296547a228e0b1e5d 2024-12-03T06:14:27,796 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/234313d5f0e54b8296547a228e0b1e5d, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T06:14:27,796 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/.tmp/C/77381b65495449f18fe54d9e7557a27e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/77381b65495449f18fe54d9e7557a27e 2024-12-03T06:14:27,800 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/77381b65495449f18fe54d9e7557a27e, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T06:14:27,800 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5535461f8d6a19172fe21e7d7b6b77aa in 1282ms, sequenceid=430, compaction requested=true 2024-12-03T06:14:27,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/d1738c6b6eb04ee5a8047f299e4b481e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a6c13525f3c54bbdb50c847cc36e556d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b37e3e5d8ef847879ebbe41f31a78d79, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/9b5549ca6e114ca99221e7e03cbbb7d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/7e18792a95f247dfa3f06f4996947426, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6326a11b11bd4761b0f4b273d3c369b7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/91a4cca0509b41f98eca641d5cb9839b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3] to archive 2024-12-03T06:14:27,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b25c5ee99f604c39a8ee27ffb5db2284 2024-12-03T06:14:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/2111573a0cae4152aa1d8bd5be8310cc 2024-12-03T06:14:27,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/248a681ab2ba42b59c9232101b0b3c14 2024-12-03T06:14:27,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/d1738c6b6eb04ee5a8047f299e4b481e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/d1738c6b6eb04ee5a8047f299e4b481e 2024-12-03T06:14:27,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e4373b1cfb144205aac933312e082477 2024-12-03T06:14:27,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/c8e6fe3e8cec4bcb98137b91fe240884 2024-12-03T06:14:27,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/3a04eb171b394d5c9bdc47a24bbcb5e2 2024-12-03T06:14:27,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a6c13525f3c54bbdb50c847cc36e556d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a6c13525f3c54bbdb50c847cc36e556d 2024-12-03T06:14:27,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e6a07e15f7dc4e37850b025d3de8aeaa 2024-12-03T06:14:27,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b37e3e5d8ef847879ebbe41f31a78d79 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b37e3e5d8ef847879ebbe41f31a78d79 2024-12-03T06:14:27,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bac38f67555a41fb9d2bd0210873db6d 2024-12-03T06:14:27,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bce2ede5b77d405882bc85464a2c339b 2024-12-03T06:14:27,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/9b5549ca6e114ca99221e7e03cbbb7d6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/9b5549ca6e114ca99221e7e03cbbb7d6 2024-12-03T06:14:27,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/47fd9329b0e74b13a2ee333d518c7f72 2024-12-03T06:14:27,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/bf1ae9514faf4638881e9007b0bc05de 2024-12-03T06:14:27,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/7e18792a95f247dfa3f06f4996947426 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/7e18792a95f247dfa3f06f4996947426 2024-12-03T06:14:27,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/89bae42ba81d48978389dd6b7aa8611c 2024-12-03T06:14:27,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b58cdb41964e4380847f2e1714590d53 2024-12-03T06:14:27,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/aea44c8baba443558aca73d7ecfe3b23 2024-12-03T06:14:27,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6326a11b11bd4761b0f4b273d3c369b7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6326a11b11bd4761b0f4b273d3c369b7 2024-12-03T06:14:27,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/80258f8f7e4d46c397d1d4f31537aab8 2024-12-03T06:14:27,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/6a49c0470d9148f592e2ae58237de4a7 2024-12-03T06:14:27,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/205c493caf5c48d6b71aceef4eeef484 2024-12-03T06:14:27,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/fdabecba39ae4bfe929135588f468d99 2024-12-03T06:14:27,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/e3a9608b7f8d45199883cdd9b8a26240 2024-12-03T06:14:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/91a4cca0509b41f98eca641d5cb9839b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/91a4cca0509b41f98eca641d5cb9839b 2024-12-03T06:14:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/b67f235a7c5342b59e41aedbb993c6ab 2024-12-03T06:14:27,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/20d043e73e334e8ba3009a3cea7a3a95 2024-12-03T06:14:27,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/5ad3ce81e286465cb6f4e2a2829d02a3 2024-12-03T06:14:27,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/2318896b52fa4a37a643a4b242c0d116, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/082a02381ceb43f098051d6199693545, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/04e0e9b731174784b2991ae55d1439bc, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5c1a6de412984ce0b2cd8206ee2debcb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/aaf0f9c0402a45b8aa1a0b24f3893d95, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a46ce579e7fb48689e6674eb09ad692c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/98eec2feebd14e47816022819ead50e6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12] to archive 2024-12-03T06:14:27,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:27,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4e5a755ba3354251889c05d6c1510387 2024-12-03T06:14:27,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/e27587f66ae84b12b3c9383cdf78c862 2024-12-03T06:14:27,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/2318896b52fa4a37a643a4b242c0d116 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/2318896b52fa4a37a643a4b242c0d116 2024-12-03T06:14:27,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/ef19698c1d5f4e4682693da4df3e45e8 2024-12-03T06:14:27,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9e654c0ba86c40f999b135ec80dfa419 2024-12-03T06:14:27,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a07e94a5bf394cb7b51960cfb8aa5451 2024-12-03T06:14:27,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/082a02381ceb43f098051d6199693545 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/082a02381ceb43f098051d6199693545 2024-12-03T06:14:27,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/efff96ec27fa4aab8fd9caf2a95a6354 2024-12-03T06:14:27,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/57e2ab634ba04f6396248c142594ad1d 2024-12-03T06:14:27,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/04e0e9b731174784b2991ae55d1439bc to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/04e0e9b731174784b2991ae55d1439bc 2024-12-03T06:14:27,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c288044439a549adadecd6b397d2f044 2024-12-03T06:14:27,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/8f376ac67b6a405bb7eb8c1e0e62ea62 2024-12-03T06:14:27,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5c1a6de412984ce0b2cd8206ee2debcb to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5c1a6de412984ce0b2cd8206ee2debcb 2024-12-03T06:14:27,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/574a82a90b304be5bb56ba72a121eb2a 2024-12-03T06:14:27,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/283c9f600e9c488588ca403f78807674 2024-12-03T06:14:27,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/aaf0f9c0402a45b8aa1a0b24f3893d95 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/aaf0f9c0402a45b8aa1a0b24f3893d95 2024-12-03T06:14:27,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/fe559d05834d42868e3d23959064313e 2024-12-03T06:14:27,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/52c85d2e49844cb6bc0de5cd396fb515 2024-12-03T06:14:27,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/7e373577a64648b4ad595d29d656bf72 2024-12-03T06:14:27,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a46ce579e7fb48689e6674eb09ad692c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/a46ce579e7fb48689e6674eb09ad692c 2024-12-03T06:14:27,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1f0c546d62f2465da0ba4904c9cb338d 2024-12-03T06:14:27,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/693a24343f5e42c590845b35e57d2e39 2024-12-03T06:14:27,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/5fd789f3476f4ffba97702a02b2697d9 2024-12-03T06:14:27,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/4ce830e8005f460481ee8635f5e2205d 2024-12-03T06:14:27,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/98eec2feebd14e47816022819ead50e6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/98eec2feebd14e47816022819ead50e6 2024-12-03T06:14:27,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/9bc443c3ca4047da96f404ef3a82d804 2024-12-03T06:14:27,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/dea710ff9d9c4001a32c963ab5c8d7e7 2024-12-03T06:14:27,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/b5e89d1b3f48447b85e4e5e7afcbdb01 2024-12-03T06:14:27,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/1800d42fd68e4d1a9bac73a0063cea12 2024-12-03T06:14:27,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/3fc6c61d65c345b680926d1eb52fb6e4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f2991f24616e4cdaab300185fa2a4280, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e06612314a3f4a9aace26d2e335ddb1e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b36ca060afaa4eb4b844ea60ad0ad631, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/524fe359962a4eae825158a17e863de9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/93ee0fe0af6b4402886213edebb8bbec, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9] to archive 2024-12-03T06:14:27,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:14:27,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/60cc98d4536c416e80ba9f99cef45641 2024-12-03T06:14:27,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/a6bd9ac95725443db4efb4001b3879a2 2024-12-03T06:14:27,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/3fc6c61d65c345b680926d1eb52fb6e4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/3fc6c61d65c345b680926d1eb52fb6e4 2024-12-03T06:14:27,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/87e3268d840146cb8565f2d6769c2c93 2024-12-03T06:14:27,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b447cacdfe4740e392008139b2a4f88a 2024-12-03T06:14:27,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c30c64a3fff149b3ac8fe880ea0aeede 2024-12-03T06:14:27,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f2991f24616e4cdaab300185fa2a4280 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f2991f24616e4cdaab300185fa2a4280 2024-12-03T06:14:27,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/5cfbacbd18e447838659b6cae29dd9f3 2024-12-03T06:14:27,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/bb9d8700e4784cf79ec2e93d129c9cd2 2024-12-03T06:14:27,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/9d1cf5ba797c4e6294209e4bafe37f1b 2024-12-03T06:14:27,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e06612314a3f4a9aace26d2e335ddb1e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e06612314a3f4a9aace26d2e335ddb1e 2024-12-03T06:14:27,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/cff7fa7040b9479bba6c6702130aa326 2024-12-03T06:14:27,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c6623f8620984e38b8808f9c93737239 2024-12-03T06:14:27,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c2fb49b8f13c409fb90d526d196347ca 2024-12-03T06:14:27,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b36ca060afaa4eb4b844ea60ad0ad631 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b36ca060afaa4eb4b844ea60ad0ad631 2024-12-03T06:14:27,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/8aef348174ba4afd809af52557b50155 2024-12-03T06:14:27,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/fdbf2cd1b35a46debe35a41fce133bbd 2024-12-03T06:14:27,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/32b98ae3d50b4078834164ffbf60721a 2024-12-03T06:14:27,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/524fe359962a4eae825158a17e863de9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/524fe359962a4eae825158a17e863de9 2024-12-03T06:14:27,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea68c28f59f54270aea6aaeba1b78a1a 2024-12-03T06:14:27,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e45812c4340141f1a40a338417f99b82 2024-12-03T06:14:27,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/ea895abd03164656aa3fb0fcd24826ca 2024-12-03T06:14:27,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/2d5161ff408541a582218e819d949721 2024-12-03T06:14:27,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/93ee0fe0af6b4402886213edebb8bbec to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/93ee0fe0af6b4402886213edebb8bbec 2024-12-03T06:14:27,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/69486bf1d449442fad7ab9782b5ed782 2024-12-03T06:14:27,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/e5bdcff2f64447e89ae6788d3282472e 2024-12-03T06:14:27,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/f413e786ba9f4362b85884c0d1591070 2024-12-03T06:14:27,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/312a309461be4d68b02ea232482609c9 2024-12-03T06:14:27,887 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/recovered.edits/433.seqid, newMaxSeqId=433, maxSeqId=1 2024-12-03T06:14:27,888 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. 2024-12-03T06:14:27,888 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 5535461f8d6a19172fe21e7d7b6b77aa: Waiting for close lock at 1733206466517Running coprocessor pre-close hooks at 1733206466517Disabling compacts and flushes for region at 1733206466517Disabling writes for close at 1733206466518 (+1 ms)Obtaining lock to block concurrent updates at 1733206466518Preparing flush snapshotting stores in 5535461f8d6a19172fe21e7d7b6b77aa at 1733206466518Finished memstore snapshotting TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733206466518Flushing stores of TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa. at 1733206466519 (+1 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/A: creating writer at 1733206466519Flushing 5535461f8d6a19172fe21e7d7b6b77aa/A: appending metadata at 1733206466522 (+3 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/A: closing flushed file at 1733206466522Flushing 5535461f8d6a19172fe21e7d7b6b77aa/B: creating writer at 1733206466930 (+408 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/B: appending metadata at 1733206466934 (+4 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/B: closing flushed file at 1733206466934Flushing 5535461f8d6a19172fe21e7d7b6b77aa/C: creating writer at 1733206467342 (+408 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/C: appending metadata at 1733206467344 (+2 ms)Flushing 5535461f8d6a19172fe21e7d7b6b77aa/C: closing flushed file at 1733206467344Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76e60901: reopening flushed file at 1733206467788 (+444 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b6c957a: reopening flushed file at 1733206467792 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11991530: reopening flushed file at 1733206467796 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5535461f8d6a19172fe21e7d7b6b77aa in 1282ms, sequenceid=430, compaction requested=true at 1733206467800 (+4 ms)Writing region close event to WAL at 1733206467885 (+85 ms)Running coprocessor post-close hooks at 1733206467888 (+3 ms)Closed at 1733206467888 2024-12-03T06:14:27,890 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:27,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=5535461f8d6a19172fe21e7d7b6b77aa, regionState=CLOSED 2024-12-03T06:14:27,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:27,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-03T06:14:27,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 5535461f8d6a19172fe21e7d7b6b77aa, server=122cddc95ec8,35255,1733206381424 in 1.5280 sec 2024-12-03T06:14:27,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T06:14:27,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5535461f8d6a19172fe21e7d7b6b77aa, UNASSIGN in 1.5340 sec 2024-12-03T06:14:27,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-03T06:14:27,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-12-03T06:14:27,898 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206467898"}]},"ts":"1733206467898"} 2024-12-03T06:14:27,899 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:14:27,899 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:14:27,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5470 sec 2024-12-03T06:14:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T06:14:28,500 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:28,501 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:14:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-12-03T06:14:28,506 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:28,509 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/recovered.edits] 2024-12-03T06:14:28,513 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/13c8e714077740418f44952d681ebe11 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/13c8e714077740418f44952d681ebe11 2024-12-03T06:14:28,515 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a52f2f0c8f0d41ac9390c7a10cc4a5b6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/a52f2f0c8f0d41ac9390c7a10cc4a5b6 2024-12-03T06:14:28,517 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/f1cdd41c9f1c4397ae6795b841a5ea69 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/A/f1cdd41c9f1c4397ae6795b841a5ea69 2024-12-03T06:14:28,520 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/234313d5f0e54b8296547a228e0b1e5d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/234313d5f0e54b8296547a228e0b1e5d 2024-12-03T06:14:28,521 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c6ddffe44e98450a9c3fc3ddfe48b370 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/c6ddffe44e98450a9c3fc3ddfe48b370 2024-12-03T06:14:28,522 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/f866bb731a7548f1ae4dfaf808918bf3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/B/f866bb731a7548f1ae4dfaf808918bf3 2024-12-03T06:14:28,525 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/77381b65495449f18fe54d9e7557a27e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/77381b65495449f18fe54d9e7557a27e 2024-12-03T06:14:28,526 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b3131726ee8f47449c97624f92a7aaf0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/b3131726ee8f47449c97624f92a7aaf0 2024-12-03T06:14:28,527 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c98a53880f4241ce81b57fdfae8326a2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/C/c98a53880f4241ce81b57fdfae8326a2 2024-12-03T06:14:28,530 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/recovered.edits/433.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa/recovered.edits/433.seqid 2024-12-03T06:14:28,531 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/5535461f8d6a19172fe21e7d7b6b77aa 2024-12-03T06:14:28,531 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:14:28,533 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,535 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:14:28,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:14:28,538 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,538 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:14:28,538 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206468538"}]},"ts":"9223372036854775807"} 2024-12-03T06:14:28,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:14:28,540 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5535461f8d6a19172fe21e7d7b6b77aa, NAME => 'TestAcidGuarantees,,1733206441882.5535461f8d6a19172fe21e7d7b6b77aa.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:14:28,540 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:14:28,540 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206468540"}]},"ts":"9223372036854775807"} 2024-12-03T06:14:28,542 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:14:28,543 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-03T06:14:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-12-03T06:14:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:14:28,612 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:28,627 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 252), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=11582 (was 12041) 2024-12-03T06:14:28,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=11582 2024-12-03T06:14:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:14:28,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:28,639 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:14:28,640 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:28,640 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-12-03T06:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:28,640 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:14:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742177_1353 (size=1061) 2024-12-03T06:14:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:29,052 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:14:29,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742178_1354 (size=53) 2024-12-03T06:14:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:29,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:14:29,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c675065cc3f30321d4b6a3915fdea885, disabling compactions & flushes 2024-12-03T06:14:29,460 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. after waiting 0 ms 2024-12-03T06:14:29,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,460 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c675065cc3f30321d4b6a3915fdea885: Waiting for close lock at 1733206469459Disabling compacts and flushes for region at 1733206469459Disabling writes for close at 1733206469460 (+1 ms)Writing region close event to WAL at 1733206469460Closed at 1733206469460 2024-12-03T06:14:29,463 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:14:29,463 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206469463"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206469463"}]},"ts":"1733206469463"} 2024-12-03T06:14:29,468 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:14:29,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:14:29,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206469470"}]},"ts":"1733206469470"} 2024-12-03T06:14:29,471 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:14:29,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, ASSIGN}] 2024-12-03T06:14:29,472 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, ASSIGN 2024-12-03T06:14:29,472 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:14:29,623 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:29,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, ASSIGN because future has completed 2024-12-03T06:14:29,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:14:29,768 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:14:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:29,785 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,785 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:14:29,785 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,786 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:14:29,786 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,786 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,788 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,790 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:29,791 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName A 2024-12-03T06:14:29,791 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:29,792 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:29,792 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,794 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:29,794 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName B 2024-12-03T06:14:29,794 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:29,795 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:29,795 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,796 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:29,797 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName C 2024-12-03T06:14:29,797 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:29,797 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:29,797 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,797 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,798 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,798 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,800 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,800 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,800 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:14:29,801 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,804 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:14:29,804 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened c675065cc3f30321d4b6a3915fdea885; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65428437, jitterRate=-0.025040313601493835}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:14:29,804 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:29,805 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for c675065cc3f30321d4b6a3915fdea885: Running coprocessor pre-open hook at 1733206469786Writing region info on filesystem at 1733206469786Initializing all the Stores at 1733206469787 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206469787Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206469788 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206469788Cleaning up temporary data from old regions at 1733206469800 (+12 ms)Running coprocessor post-open hooks at 1733206469804 (+4 ms)Region opened successfully at 1733206469805 (+1 ms) 2024-12-03T06:14:29,806 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., pid=101, masterSystemTime=1733206469780 2024-12-03T06:14:29,808 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,808 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:29,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:29,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:29,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-12-03T06:14:29,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 in 186 msec 2024-12-03T06:14:29,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T06:14:29,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, ASSIGN in 344 msec 2024-12-03T06:14:29,817 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:14:29,817 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206469817"}]},"ts":"1733206469817"} 2024-12-03T06:14:29,819 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:14:29,819 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:14:29,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-03T06:14:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T06:14:30,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:30,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:30,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:30,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:30,807 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428464f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:30,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11db1ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:30,811 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:30,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:30,814 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:14:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:14:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fda85e8 2024-12-03T06:14:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:14:30,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:14:30,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:14:30,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:14:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T06:14:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742179_1355 (size=1097) 2024-12-03T06:14:31,230 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T06:14:31,230 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T06:14:31,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:14:31,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE}] 2024-12-03T06:14:31,239 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE 2024-12-03T06:14:31,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE because future has completed 2024-12-03T06:14:31,242 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:14:31,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:14:31,395 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,395 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:14:31,395 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing c675065cc3f30321d4b6a3915fdea885, disabling compactions & flushes 2024-12-03T06:14:31,395 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,395 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,395 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. after waiting 0 ms 2024-12-03T06:14:31,395 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,398 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:14:31,399 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,399 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for c675065cc3f30321d4b6a3915fdea885: Waiting for close lock at 1733206471395Running coprocessor pre-close hooks at 1733206471395Disabling compacts and flushes for region at 1733206471395Disabling writes for close at 1733206471395Writing region close event to WAL at 1733206471396 (+1 ms)Running coprocessor post-close hooks at 1733206471399 (+3 ms)Closed at 1733206471399 2024-12-03T06:14:31,399 WARN [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: c675065cc3f30321d4b6a3915fdea885 to self. 2024-12-03T06:14:31,401 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,401 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=CLOSED 2024-12-03T06:14:31,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:31,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-03T06:14:31,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 in 160 msec 2024-12-03T06:14:31,405 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE; state=CLOSED, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=true 2024-12-03T06:14:31,556 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE because future has completed 2024-12-03T06:14:31,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:14:31,712 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,712 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:14:31,713 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,713 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:14:31,713 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,713 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,715 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,716 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:31,716 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName A 2024-12-03T06:14:31,717 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:31,718 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:31,718 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,719 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:31,719 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName B 2024-12-03T06:14:31,719 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:31,719 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:31,720 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,720 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:14:31,720 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c675065cc3f30321d4b6a3915fdea885 columnFamilyName C 2024-12-03T06:14:31,720 DEBUG [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:31,721 INFO [StoreOpener-c675065cc3f30321d4b6a3915fdea885-1 {}] regionserver.HStore(327): Store=c675065cc3f30321d4b6a3915fdea885/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:14:31,721 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,721 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,722 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,723 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,724 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,724 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,725 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:14:31,726 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,727 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened c675065cc3f30321d4b6a3915fdea885; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66682286, jitterRate=-0.0063565075397491455}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:14:31,727 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,727 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for c675065cc3f30321d4b6a3915fdea885: Running coprocessor pre-open hook at 1733206471713Writing region info on filesystem at 1733206471713Initializing all the Stores at 1733206471714 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206471714Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206471714Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206471714Cleaning up temporary data from old regions at 1733206471724 (+10 ms)Running coprocessor post-open hooks at 1733206471727 (+3 ms)Region opened successfully at 1733206471727 2024-12-03T06:14:31,728 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., pid=106, masterSystemTime=1733206471709 2024-12-03T06:14:31,731 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,731 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,731 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=OPEN, openSeqNum=5, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:14:31,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-12-03T06:14:31,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 in 177 msec 2024-12-03T06:14:31,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T06:14:31,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, REOPEN/MOVE in 498 msec 2024-12-03T06:14:31,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-03T06:14:31,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-12-03T06:14:31,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-12-03T06:14:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-03T06:14:31,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:31,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c42075e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,746 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1679e3cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6b056, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502d3d2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fdc5e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd96c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,754 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422b9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1382d00e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd5ed4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569d13d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c5e4fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d79187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,764 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108659b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5d83fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,768 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350a17ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,770 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690d72d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd929ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd6c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:14:31,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:14:31,777 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:14:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:14:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:14:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcb42e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:14:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:14:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:31,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:14:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2474ce54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:14:31,780 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,780 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,780 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,780 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,780 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,781 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,781 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,783 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,786 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,788 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:14:31,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:14:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:14:31,790 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:14:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:14:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-03T06:14:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:31,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:31,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:31,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:31,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:31,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd04b54f61fb4c2485925e59b36d863a_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206471797/Put/seqid=0 2024-12-03T06:14:31,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742180_1356 (size=12154) 2024-12-03T06:14:31,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206531844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206531844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206531844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206531845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206531845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:31,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:31,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:31,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206531959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206531959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206531959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206531960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206531960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:32,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206532171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:32,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd04b54f61fb4c2485925e59b36d863a_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd04b54f61fb4c2485925e59b36d863a_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:32,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0701c474c7cc485dafe7c92e6979b13b, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:32,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0701c474c7cc485dafe7c92e6979b13b is 175, key is test_row_0/A:col10/1733206471797/Put/seqid=0 2024-12-03T06:14:32,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742181_1357 (size=30955) 2024-12-03T06:14:32,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:32,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206532480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206532480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206532490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206532490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206532490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:32,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0701c474c7cc485dafe7c92e6979b13b 2024-12-03T06:14:32,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/bea77aa527e143d184be7ef13826cadd is 50, key is test_row_0/B:col10/1733206471797/Put/seqid=0 2024-12-03T06:14:32,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742182_1358 (size=12001) 2024-12-03T06:14:32,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/bea77aa527e143d184be7ef13826cadd 2024-12-03T06:14:32,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:32,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/146c3bf1389240958b2ea6d50f14fb04 is 50, key is test_row_0/C:col10/1733206471797/Put/seqid=0 2024-12-03T06:14:32,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742183_1359 (size=12001) 2024-12-03T06:14:32,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/146c3bf1389240958b2ea6d50f14fb04 2024-12-03T06:14:32,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0701c474c7cc485dafe7c92e6979b13b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b 2024-12-03T06:14:32,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b, entries=150, sequenceid=16, filesize=30.2 K 2024-12-03T06:14:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/bea77aa527e143d184be7ef13826cadd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd 2024-12-03T06:14:32,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T06:14:32,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/146c3bf1389240958b2ea6d50f14fb04 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04 2024-12-03T06:14:32,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T06:14:32,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c675065cc3f30321d4b6a3915fdea885 in 1027ms, sequenceid=16, compaction requested=false 2024-12-03T06:14:32,829 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T06:14:32,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:32,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T06:14:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:32,868 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:14:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:32,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f6f6c96176c44b5ea55976e1dc8e4f0d_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206471843/Put/seqid=0 2024-12-03T06:14:32,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742184_1360 (size=12154) 2024-12-03T06:14:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:32,934 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f6f6c96176c44b5ea55976e1dc8e4f0d_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f6c96176c44b5ea55976e1dc8e4f0d_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/13da072b324f47ca9b2eab6b51d2e079, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/13da072b324f47ca9b2eab6b51d2e079 is 175, key is test_row_0/A:col10/1733206471843/Put/seqid=0 2024-12-03T06:14:32,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742185_1361 (size=30955) 2024-12-03T06:14:32,959 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/13da072b324f47ca9b2eab6b51d2e079 2024-12-03T06:14:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/70c29f706e304937a589b9625c98268a is 50, key is test_row_0/B:col10/1733206471843/Put/seqid=0 2024-12-03T06:14:32,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:32,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742186_1362 (size=12001) 2024-12-03T06:14:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206533011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206533011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206533016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206533018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206533018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206533130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206533130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206533130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206533140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206533140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206533339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206533349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206533349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206533362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206533362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,400 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/70c29f706e304937a589b9625c98268a 2024-12-03T06:14:33,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/4f4fb5b5a552447e915c2c0858c3bb57 is 50, key is test_row_0/C:col10/1733206471843/Put/seqid=0 2024-12-03T06:14:33,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:14:33,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742187_1363 (size=12001) 2024-12-03T06:14:33,492 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/4f4fb5b5a552447e915c2c0858c3bb57 2024-12-03T06:14:33,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/13da072b324f47ca9b2eab6b51d2e079 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079 2024-12-03T06:14:33,514 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079, entries=150, sequenceid=36, filesize=30.2 K 2024-12-03T06:14:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/70c29f706e304937a589b9625c98268a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a 2024-12-03T06:14:33,523 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T06:14:33,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/4f4fb5b5a552447e915c2c0858c3bb57 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57 2024-12-03T06:14:33,542 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T06:14:33,544 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c675065cc3f30321d4b6a3915fdea885 in 676ms, sequenceid=36, compaction requested=false 2024-12-03T06:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-03T06:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-03T06:14:33,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T06:14:33,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7500 sec 2024-12-03T06:14:33,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.7590 sec 2024-12-03T06:14:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:33,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:14:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:33,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:33,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:33,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120335317adf895a49cfa6f9ffc35106623b_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742188_1364 (size=14594) 2024-12-03T06:14:33,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206533698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206533699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206533708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206533710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206533712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206533819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206533819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206533830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206533830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206533830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T06:14:33,950 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:33,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-03T06:14:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:33,952 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:33,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:33,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206534030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206534030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206534040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206534040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206534040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:34,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:34,093 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120335317adf895a49cfa6f9ffc35106623b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335317adf895a49cfa6f9ffc35106623b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:34,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/871e7c72ed3444fca4f9fd35338b2f54, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:34,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/871e7c72ed3444fca4f9fd35338b2f54 is 175, key is test_row_0/A:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:34,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742189_1365 (size=39549) 2024-12-03T06:14:34,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/871e7c72ed3444fca4f9fd35338b2f54 2024-12-03T06:14:34,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/12d446c1ceb04816a49059b033edd006 is 50, key is test_row_0/B:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:34,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742190_1366 (size=12001) 2024-12-03T06:14:34,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:34,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206534350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206534351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206534360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206534360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206534360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/12d446c1ceb04816a49059b033edd006 2024-12-03T06:14:34,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/19d721ab3d1b4268a40c44069a45c7f5 is 50, key is test_row_0/C:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:34,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742191_1367 (size=12001) 2024-12-03T06:14:34,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:34,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206534860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206534869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:34,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206534879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206534879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206534879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:34,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/19d721ab3d1b4268a40c44069a45c7f5 2024-12-03T06:14:34,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/871e7c72ed3444fca4f9fd35338b2f54 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54 2024-12-03T06:14:34,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54, entries=200, sequenceid=50, filesize=38.6 K 2024-12-03T06:14:34,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/12d446c1ceb04816a49059b033edd006 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006 2024-12-03T06:14:34,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T06:14:34,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/19d721ab3d1b4268a40c44069a45c7f5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5 2024-12-03T06:14:34,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T06:14:34,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c675065cc3f30321d4b6a3915fdea885 in 1315ms, sequenceid=50, compaction requested=true 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:34,968 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:34,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:14:34,969 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:34,970 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:34,970 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:34,970 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,970 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=99.1 K 2024-12-03T06:14:34,970 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,970 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54] 2024-12-03T06:14:34,970 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:34,970 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:34,970 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:34,971 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.2 K 2024-12-03T06:14:34,971 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting bea77aa527e143d184be7ef13826cadd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733206471797 2024-12-03T06:14:34,971 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0701c474c7cc485dafe7c92e6979b13b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733206471797 2024-12-03T06:14:34,971 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c29f706e304937a589b9625c98268a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733206471814 2024-12-03T06:14:34,971 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13da072b324f47ca9b2eab6b51d2e079, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733206471814 2024-12-03T06:14:34,972 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 871e7c72ed3444fca4f9fd35338b2f54, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473014 2024-12-03T06:14:34,972 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d446c1ceb04816a49059b033edd006, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473015 2024-12-03T06:14:34,989 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:35,000 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:35,001 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/de4eb0b735dd4ad58fc05c66197a4f5d is 50, key is test_row_0/B:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:35,002 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120342c64a33c5de4c68abb4b9e58727f6a7_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:35,004 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120342c64a33c5de4c68abb4b9e58727f6a7_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:35,004 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120342c64a33c5de4c68abb4b9e58727f6a7_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742192_1368 (size=12104) 2024-12-03T06:14:35,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742193_1369 (size=4469) 2024-12-03T06:14:35,023 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#315 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:35,024 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/3a4379010f6443abbfb18eb1430cc114 is 175, key is test_row_0/A:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:35,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T06:14:35,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:35,025 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:35,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b3eb7b2f56ba44728f4385b60c77724f_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206473710/Put/seqid=0 2024-12-03T06:14:35,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742194_1370 (size=31058) 2024-12-03T06:14:35,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742195_1371 (size=12154) 2024-12-03T06:14:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:35,429 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/de4eb0b735dd4ad58fc05c66197a4f5d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/de4eb0b735dd4ad58fc05c66197a4f5d 2024-12-03T06:14:35,436 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into de4eb0b735dd4ad58fc05c66197a4f5d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:35,436 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:35,436 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206474968; duration=0sec 2024-12-03T06:14:35,437 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:35,437 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:35,437 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:35,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:35,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:35,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:35,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.2 K 2024-12-03T06:14:35,438 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/3a4379010f6443abbfb18eb1430cc114 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114 2024-12-03T06:14:35,439 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 146c3bf1389240958b2ea6d50f14fb04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733206471797 2024-12-03T06:14:35,439 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4fb5b5a552447e915c2c0858c3bb57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733206471814 2024-12-03T06:14:35,440 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d721ab3d1b4268a40c44069a45c7f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473015 2024-12-03T06:14:35,445 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into 3a4379010f6443abbfb18eb1430cc114(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:35,445 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:35,445 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206474968; duration=0sec 2024-12-03T06:14:35,445 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:35,448 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:35,456 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b3eb7b2f56ba44728f4385b60c77724f_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b3eb7b2f56ba44728f4385b60c77724f_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a137ba6adac74880a72778cc5d9620e0, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a137ba6adac74880a72778cc5d9620e0 is 175, key is test_row_0/A:col10/1733206473710/Put/seqid=0 2024-12-03T06:14:35,461 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:35,462 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/91e4539cde8e4cb9afdf201a75849995 is 50, key is test_row_0/C:col10/1733206473015/Put/seqid=0 2024-12-03T06:14:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742196_1372 (size=30955) 2024-12-03T06:14:35,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742197_1373 (size=12104) 2024-12-03T06:14:35,531 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/91e4539cde8e4cb9afdf201a75849995 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/91e4539cde8e4cb9afdf201a75849995 2024-12-03T06:14:35,541 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 91e4539cde8e4cb9afdf201a75849995(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:35,541 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:35,541 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206474968; duration=0sec 2024-12-03T06:14:35,541 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:35,541 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:35,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:35,896 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a137ba6adac74880a72778cc5d9620e0 2024-12-03T06:14:35,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b5efcf3f4f0b497b8fd0f82891715248 is 50, key is test_row_0/B:col10/1733206473710/Put/seqid=0 2024-12-03T06:14:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206535906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206535908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:35,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206535911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:35,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:35,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206535916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:35,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206535917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:35,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742198_1374 (size=12001) 2024-12-03T06:14:36,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206536020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206536020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206536029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206536029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206536030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:36,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206536229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206536229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206536239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206536240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206536240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,339 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b5efcf3f4f0b497b8fd0f82891715248 2024-12-03T06:14:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/79b7d84c02334c4d9a32fd791f0d2d3c is 50, key is test_row_0/C:col10/1733206473710/Put/seqid=0 2024-12-03T06:14:36,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742199_1375 (size=12001) 2024-12-03T06:14:36,401 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/79b7d84c02334c4d9a32fd791f0d2d3c 2024-12-03T06:14:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a137ba6adac74880a72778cc5d9620e0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0 2024-12-03T06:14:36,419 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0, entries=150, sequenceid=68, filesize=30.2 K 2024-12-03T06:14:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b5efcf3f4f0b497b8fd0f82891715248 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248 2024-12-03T06:14:36,425 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T06:14:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/79b7d84c02334c4d9a32fd791f0d2d3c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c 2024-12-03T06:14:36,431 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T06:14:36,432 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c675065cc3f30321d4b6a3915fdea885 in 1407ms, sequenceid=68, compaction requested=false 2024-12-03T06:14:36,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:36,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-03T06:14:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-03T06:14:36,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T06:14:36,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4820 sec 2024-12-03T06:14:36,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.4930 sec 2024-12-03T06:14:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:36,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:36,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cae0cb169be64f1bb1e4cc37efcb5014_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742200_1376 (size=12154) 2024-12-03T06:14:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206536567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206536570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206536570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206536572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206536573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206536679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206536680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206536680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206536689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206536689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206536890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206536890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206536890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206536899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206536899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:36,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:36,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cae0cb169be64f1bb1e4cc37efcb5014_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cae0cb169be64f1bb1e4cc37efcb5014_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:36,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/72caaef430184a6cba36d915667d33d7, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:36,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/72caaef430184a6cba36d915667d33d7 is 175, key is test_row_0/A:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:36,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742201_1377 (size=30955) 2024-12-03T06:14:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206537200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206537200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206537210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206537219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206537219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/72caaef430184a6cba36d915667d33d7 2024-12-03T06:14:37,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/973bd903b30a4734920393df3af26fac is 50, key is test_row_0/B:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:37,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742202_1378 (size=12001) 2024-12-03T06:14:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206537710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206537710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206537729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206537729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206537730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:37,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/973bd903b30a4734920393df3af26fac 2024-12-03T06:14:37,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/59acb46e16074fcdadf09cc545a46aae is 50, key is test_row_0/C:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:37,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742203_1379 (size=12001) 2024-12-03T06:14:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T06:14:38,120 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:38,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-03T06:14:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T06:14:38,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:38,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:38,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T06:14:38,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/59acb46e16074fcdadf09cc545a46aae 2024-12-03T06:14:38,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/72caaef430184a6cba36d915667d33d7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7 2024-12-03T06:14:38,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7, entries=150, sequenceid=88, filesize=30.2 K 2024-12-03T06:14:38,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/973bd903b30a4734920393df3af26fac as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac 2024-12-03T06:14:38,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T06:14:38,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/59acb46e16074fcdadf09cc545a46aae as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae 2024-12-03T06:14:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T06:14:38,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c675065cc3f30321d4b6a3915fdea885 in 1703ms, sequenceid=88, compaction requested=true 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:38,249 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:38,249 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:38,252 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:38,252 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:38,252 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:38,252 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:38,252 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:38,252 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:38,252 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=90.8 K 2024-12-03T06:14:38,252 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/de4eb0b735dd4ad58fc05c66197a4f5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.3 K 2024-12-03T06:14:38,252 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7] 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting de4eb0b735dd4ad58fc05c66197a4f5d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473015 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4379010f6443abbfb18eb1430cc114, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473015 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b5efcf3f4f0b497b8fd0f82891715248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733206473695 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a137ba6adac74880a72778cc5d9620e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733206473695 2024-12-03T06:14:38,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 973bd903b30a4734920393df3af26fac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:38,254 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72caaef430184a6cba36d915667d33d7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:38,258 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:38,258 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:38,259 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/4793de6e63c44fd59c65e13414d37f1e is 50, key is test_row_0/B:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:38,261 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412030a5979f502e142068e908117208a3cf9_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:38,261 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412030a5979f502e142068e908117208a3cf9_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:38,262 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030a5979f502e142068e908117208a3cf9_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:38,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742204_1380 (size=12207) 2024-12-03T06:14:38,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-03T06:14:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:38,275 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:14:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:38,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:38,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:38,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:38,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:38,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:38,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742205_1381 (size=4469) 2024-12-03T06:14:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037633b12e0e5e47f3a44b21631ff58c60_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206476571/Put/seqid=0 2024-12-03T06:14:38,290 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#325 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:38,291 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e28da75366d34c19815720e4c5e5972e is 175, key is test_row_0/A:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742207_1383 (size=31161) 2024-12-03T06:14:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742206_1382 (size=12154) 2024-12-03T06:14:38,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:38,311 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037633b12e0e5e47f3a44b21631ff58c60_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037633b12e0e5e47f3a44b21631ff58c60_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:38,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/5259304f8166414584c1c7c80bd24998, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:38,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/5259304f8166414584c1c7c80bd24998 is 175, key is test_row_0/A:col10/1733206476571/Put/seqid=0 2024-12-03T06:14:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742208_1384 (size=30955) 2024-12-03T06:14:38,329 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/5259304f8166414584c1c7c80bd24998 2024-12-03T06:14:38,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/f267968d3d1346b7bb9874262cbf1b0d is 50, key is test_row_0/B:col10/1733206476571/Put/seqid=0 2024-12-03T06:14:38,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742209_1385 (size=12001) 2024-12-03T06:14:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T06:14:38,675 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/4793de6e63c44fd59c65e13414d37f1e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/4793de6e63c44fd59c65e13414d37f1e 2024-12-03T06:14:38,679 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into 4793de6e63c44fd59c65e13414d37f1e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:38,679 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:38,679 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206478249; duration=0sec 2024-12-03T06:14:38,679 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:38,679 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:38,679 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:38,680 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:38,680 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:38,680 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:38,680 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/91e4539cde8e4cb9afdf201a75849995, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.3 K 2024-12-03T06:14:38,680 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e4539cde8e4cb9afdf201a75849995, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733206473015 2024-12-03T06:14:38,681 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b7d84c02334c4d9a32fd791f0d2d3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733206473695 2024-12-03T06:14:38,682 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 59acb46e16074fcdadf09cc545a46aae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:38,688 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:38,688 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/43d0e8b7cc714dfaa3374e0f06c64ec8 is 50, key is test_row_0/C:col10/1733206476544/Put/seqid=0 2024-12-03T06:14:38,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742210_1386 (size=12207) 2024-12-03T06:14:38,709 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e28da75366d34c19815720e4c5e5972e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e 2024-12-03T06:14:38,714 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into e28da75366d34c19815720e4c5e5972e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:38,714 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:38,714 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206478249; duration=0sec 2024-12-03T06:14:38,714 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:38,714 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:38,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:38,749 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/f267968d3d1346b7bb9874262cbf1b0d 2024-12-03T06:14:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T06:14:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/25f7258fa3014763b26ec3f00eee741e is 50, key is test_row_0/C:col10/1733206476571/Put/seqid=0 2024-12-03T06:14:38,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742211_1387 (size=12001) 2024-12-03T06:14:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206538755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206538757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206538758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206538765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206538771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206538879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206538879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206538879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206538879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206538890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206539100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206539100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,105 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/43d0e8b7cc714dfaa3374e0f06c64ec8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/43d0e8b7cc714dfaa3374e0f06c64ec8 2024-12-03T06:14:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206539100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206539100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,113 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 43d0e8b7cc714dfaa3374e0f06c64ec8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:39,113 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:39,114 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206478249; duration=0sec 2024-12-03T06:14:39,114 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:39,114 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:39,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206539109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,163 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/25f7258fa3014763b26ec3f00eee741e 2024-12-03T06:14:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/5259304f8166414584c1c7c80bd24998 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998 2024-12-03T06:14:39,170 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998, entries=150, sequenceid=103, filesize=30.2 K 2024-12-03T06:14:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/f267968d3d1346b7bb9874262cbf1b0d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d 2024-12-03T06:14:39,174 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T06:14:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/25f7258fa3014763b26ec3f00eee741e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e 2024-12-03T06:14:39,177 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T06:14:39,178 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c675065cc3f30321d4b6a3915fdea885 in 903ms, sequenceid=103, compaction requested=false 2024-12-03T06:14:39,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:39,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-03T06:14:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-03T06:14:39,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T06:14:39,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0570 sec 2024-12-03T06:14:39,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0610 sec 2024-12-03T06:14:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T06:14:39,260 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:39,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-03T06:14:39,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:39,262 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:39,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:39,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:14:39,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:39,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:39,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:39,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203704892b72080480f960692179088232b_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:39,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742212_1388 (size=14594) 2024-12-03T06:14:39,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:39,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203704892b72080480f960692179088232b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203704892b72080480f960692179088232b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:39,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e95caf6c5fa042e6b4b606b1a7905cd3, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:39,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e95caf6c5fa042e6b4b606b1a7905cd3 is 175, key is test_row_0/A:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742213_1389 (size=39549) 2024-12-03T06:14:39,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206539433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206539434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206539436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206539437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206539442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206539550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206539550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206539550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206539550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206539550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:39,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:39,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206539759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206539759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206539759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206539759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206539759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:39,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e95caf6c5fa042e6b4b606b1a7905cd3 2024-12-03T06:14:39,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/57d324dba4f74143b3eec5fa95633067 is 50, key is test_row_0/B:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:39,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742214_1390 (size=12001) 2024-12-03T06:14:39,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/57d324dba4f74143b3eec5fa95633067 2024-12-03T06:14:39,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/097d8f299a5648e78a53f90d7d12b615 is 50, key is test_row_0/C:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:39,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742215_1391 (size=12001) 2024-12-03T06:14:39,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:39,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:39,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:39,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:40,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206540070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206540070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206540070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206540070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206540070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:40,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/097d8f299a5648e78a53f90d7d12b615 2024-12-03T06:14:40,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e95caf6c5fa042e6b4b606b1a7905cd3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3 2024-12-03T06:14:40,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3, entries=200, sequenceid=124, filesize=38.6 K 2024-12-03T06:14:40,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/57d324dba4f74143b3eec5fa95633067 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067 2024-12-03T06:14:40,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067, entries=150, sequenceid=124, filesize=11.7 K 2024-12-03T06:14:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/097d8f299a5648e78a53f90d7d12b615 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615 2024-12-03T06:14:40,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615, entries=150, sequenceid=124, filesize=11.7 K 2024-12-03T06:14:40,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c675065cc3f30321d4b6a3915fdea885 in 861ms, sequenceid=124, compaction requested=true 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:40,274 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:40,274 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:40,275 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,275 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,275 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/4793de6e63c44fd59c65e13414d37f1e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.4 K 2024-12-03T06:14:40,275 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=99.3 K 2024-12-03T06:14:40,275 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3] 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28da75366d34c19815720e4c5e5972e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:40,275 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4793de6e63c44fd59c65e13414d37f1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:40,276 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5259304f8166414584c1c7c80bd24998, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733206476567 2024-12-03T06:14:40,276 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f267968d3d1346b7bb9874262cbf1b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733206476567 2024-12-03T06:14:40,276 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e95caf6c5fa042e6b4b606b1a7905cd3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478749 2024-12-03T06:14:40,276 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d324dba4f74143b3eec5fa95633067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478763 2024-12-03T06:14:40,281 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:40,281 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#333 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:40,282 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/49190b2f47394b71bffe15affe490860 is 50, key is test_row_0/B:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:40,283 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120349cfc17388a74e39b4fc8ef5cf779576_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:40,284 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120349cfc17388a74e39b4fc8ef5cf779576_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:40,284 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120349cfc17388a74e39b4fc8ef5cf779576_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:40,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742216_1392 (size=12309) 2024-12-03T06:14:40,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742217_1393 (size=4469) 2024-12-03T06:14:40,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,331 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:40,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120310a331382bb849daa42a0f91660cdd03_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206479436/Put/seqid=0 2024-12-03T06:14:40,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742218_1394 (size=12304) 2024-12-03T06:14:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:40,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206540620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206540621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206540621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206540635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206540637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,691 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/49190b2f47394b71bffe15affe490860 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/49190b2f47394b71bffe15affe490860 2024-12-03T06:14:40,692 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:40,692 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/1dcd063a526744d39cee5be9f8bebb5d is 175, key is test_row_0/A:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:40,695 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into 49190b2f47394b71bffe15affe490860(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:40,695 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:40,695 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206480274; duration=0sec 2024-12-03T06:14:40,695 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:40,695 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:40,695 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:40,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742219_1395 (size=31263) 2024-12-03T06:14:40,696 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:40,696 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:40,696 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:40,696 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/43d0e8b7cc714dfaa3374e0f06c64ec8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.4 K 2024-12-03T06:14:40,696 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d0e8b7cc714dfaa3374e0f06c64ec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733206475913 2024-12-03T06:14:40,697 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f7258fa3014763b26ec3f00eee741e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733206476567 2024-12-03T06:14:40,697 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 097d8f299a5648e78a53f90d7d12b615, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478763 2024-12-03T06:14:40,701 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:40,701 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/92e7b4b2efa94d1e9edde951a8b5fa76 is 50, key is test_row_0/C:col10/1733206478769/Put/seqid=0 2024-12-03T06:14:40,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742220_1396 (size=12309) 2024-12-03T06:14:40,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:40,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206540740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,745 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120310a331382bb849daa42a0f91660cdd03_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120310a331382bb849daa42a0f91660cdd03_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/783c5e55be284277961857986d3d4909, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:40,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/783c5e55be284277961857986d3d4909 is 175, key is test_row_0/A:col10/1733206479436/Put/seqid=0 2024-12-03T06:14:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742221_1397 (size=31105) 2024-12-03T06:14:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206540750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206540750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206540759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206540759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206540950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206540959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206540959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206540969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:40,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206540979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,100 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/1dcd063a526744d39cee5be9f8bebb5d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d 2024-12-03T06:14:41,103 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into 1dcd063a526744d39cee5be9f8bebb5d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:41,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:41,103 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206480274; duration=0sec 2024-12-03T06:14:41,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:41,103 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:41,116 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/92e7b4b2efa94d1e9edde951a8b5fa76 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/92e7b4b2efa94d1e9edde951a8b5fa76 2024-12-03T06:14:41,119 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 92e7b4b2efa94d1e9edde951a8b5fa76(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:41,119 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:41,119 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206480274; duration=0sec 2024-12-03T06:14:41,120 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:41,120 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:41,150 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/783c5e55be284277961857986d3d4909 2024-12-03T06:14:41,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/a97117076c834286bd299e3fa2e9a9e3 is 50, key is test_row_0/B:col10/1733206479436/Put/seqid=0 2024-12-03T06:14:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742222_1398 (size=12151) 2024-12-03T06:14:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206541260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206541270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206541270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206541279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206541289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:41,559 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/a97117076c834286bd299e3fa2e9a9e3 2024-12-03T06:14:41,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/fab8c0b071ce4dbb864607416f7c1d04 is 50, key is test_row_0/C:col10/1733206479436/Put/seqid=0 2024-12-03T06:14:41,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742223_1399 (size=12151) 2024-12-03T06:14:41,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206541769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206541779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206541779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206541790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206541799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:41,969 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/fab8c0b071ce4dbb864607416f7c1d04 2024-12-03T06:14:41,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/783c5e55be284277961857986d3d4909 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909 2024-12-03T06:14:41,985 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909, entries=150, sequenceid=138, filesize=30.4 K 2024-12-03T06:14:41,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/a97117076c834286bd299e3fa2e9a9e3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3 2024-12-03T06:14:41,989 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T06:14:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/fab8c0b071ce4dbb864607416f7c1d04 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04 2024-12-03T06:14:41,993 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T06:14:41,994 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c675065cc3f30321d4b6a3915fdea885 in 1663ms, sequenceid=138, compaction requested=false 2024-12-03T06:14:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-03T06:14:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-03T06:14:41,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T06:14:41,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7340 sec 2024-12-03T06:14:42,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.7380 sec 2024-12-03T06:14:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:42,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206542817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206542819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,826 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206542824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206542825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206542825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f46adc6bcded4545a6d94c9b8c9b6aa6_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:42,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742224_1400 (size=17284) 2024-12-03T06:14:42,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206542929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206542929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206542939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206542940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206543139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206543139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206543149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206543150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:43,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f46adc6bcded4545a6d94c9b8c9b6aa6_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f46adc6bcded4545a6d94c9b8c9b6aa6_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:43,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/bcd525c4c3ba4b2ebb0444036b0c4f7b, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:43,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/bcd525c4c3ba4b2ebb0444036b0c4f7b is 175, key is test_row_0/A:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:43,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742225_1401 (size=48389) 2024-12-03T06:14:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T06:14:43,420 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:43,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-03T06:14:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:43,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:43,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:43,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:43,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206543449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206543449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206543459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206543470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:43,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T06:14:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/bcd525c4c3ba4b2ebb0444036b0c4f7b 2024-12-03T06:14:43,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/5bf16acf594d4b0ba30d178ed4faf939 is 50, key is test_row_0/B:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:43,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742226_1402 (size=12151) 2024-12-03T06:14:43,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T06:14:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:43,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T06:14:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:43,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206543960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206543960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206543970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:43,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206543979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:44,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T06:14:44,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:44,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:44,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/5bf16acf594d4b0ba30d178ed4faf939 2024-12-03T06:14:44,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/01ad89bbb4534f72a3f37ca76ceff913 is 50, key is test_row_0/C:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:44,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742227_1403 (size=12151) 2024-12-03T06:14:44,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/01ad89bbb4534f72a3f37ca76ceff913 2024-12-03T06:14:44,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/bcd525c4c3ba4b2ebb0444036b0c4f7b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b 2024-12-03T06:14:44,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b, entries=250, sequenceid=161, filesize=47.3 K 2024-12-03T06:14:44,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/5bf16acf594d4b0ba30d178ed4faf939 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939 2024-12-03T06:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939, entries=150, sequenceid=161, filesize=11.9 K 2024-12-03T06:14:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/01ad89bbb4534f72a3f37ca76ceff913 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913 2024-12-03T06:14:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913, entries=150, sequenceid=161, filesize=11.9 K 2024-12-03T06:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c675065cc3f30321d4b6a3915fdea885 in 1307ms, sequenceid=161, compaction requested=true 2024-12-03T06:14:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:44,115 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,115 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,116 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:44,116 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:44,116 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,117 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=108.2 K 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,117 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,117 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b] 2024-12-03T06:14:44,117 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:44,117 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,117 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,117 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/49190b2f47394b71bffe15affe490860, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.8 K 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,117 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 49190b2f47394b71bffe15affe490860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478763 2024-12-03T06:14:44,117 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dcd063a526744d39cee5be9f8bebb5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478763 2024-12-03T06:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,118 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a97117076c834286bd299e3fa2e9a9e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206479431 2024-12-03T06:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,118 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 783c5e55be284277961857986d3d4909, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206479431 2024-12-03T06:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,118 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf16acf594d4b0ba30d178ed4faf939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480634 2024-12-03T06:14:44,118 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd525c4c3ba4b2ebb0444036b0c4f7b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480619 2024-12-03T06:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,136 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,136 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/0de03a21a6d748e2baadd7e99b0f1de5 is 50, key is test_row_0/B:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,147 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,152 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203dfc624aa659c4e38b6871539587645a4_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,154 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203dfc624aa659c4e38b6871539587645a4_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:44,154 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dfc624aa659c4e38b6871539587645a4_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742228_1404 (size=12561) 2024-12-03T06:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742229_1405 (size=4469) 2024-12-03T06:14:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,182 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#343 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,183 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/c772fd8040ee40ec9da13055ea27db72 is 175, key is test_row_0/A:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T06:14:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,188 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742230_1406 (size=31515) 2024-12-03T06:14:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c4a76888d9f4d3ebd85dfc4c3a3156d_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206482823/Put/seqid=0 2024-12-03T06:14:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,206 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/c772fd8040ee40ec9da13055ea27db72 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72 2024-12-03T06:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,213 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into c772fd8040ee40ec9da13055ea27db72(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:44,213 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,213 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206484115; duration=0sec 2024-12-03T06:14:44,213 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:44,213 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:44,213 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,215 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:44,215 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:44,215 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:44,215 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/92e7b4b2efa94d1e9edde951a8b5fa76, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=35.8 K 2024-12-03T06:14:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,216 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e7b4b2efa94d1e9edde951a8b5fa76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733206478763 2024-12-03T06:14:44,217 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab8c0b071ce4dbb864607416f7c1d04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733206479431 2024-12-03T06:14:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,218 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01ad89bbb4534f72a3f37ca76ceff913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480634 2024-12-03T06:14:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,230 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,231 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/959b28c6c9224445bc7ee5fc4c5847f4 is 50, key is test_row_0/C:col10/1733206482793/Put/seqid=0 2024-12-03T06:14:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742231_1407 (size=9814) 2024-12-03T06:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742232_1408 (size=12561) 2024-12-03T06:14:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,584 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/0de03a21a6d748e2baadd7e99b0f1de5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0de03a21a6d748e2baadd7e99b0f1de5 2024-12-03T06:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into 0de03a21a6d748e2baadd7e99b0f1de5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,588 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:44,588 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206484115; duration=0sec 2024-12-03T06:14:44,588 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:44,588 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,659 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c4a76888d9f4d3ebd85dfc4c3a3156d_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c4a76888d9f4d3ebd85dfc4c3a3156d_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ff50398d402645819f953f3fa0180319, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ff50398d402645819f953f3fa0180319 is 175, key is test_row_0/A:col10/1733206482823/Put/seqid=0 2024-12-03T06:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742233_1409 (size=22461) 2024-12-03T06:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,688 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/959b28c6c9224445bc7ee5fc4c5847f4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/959b28c6c9224445bc7ee5fc4c5847f4 2024-12-03T06:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,692 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 959b28c6c9224445bc7ee5fc4c5847f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:44,692 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:44,692 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206484116; duration=0sec 2024-12-03T06:14:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,692 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:44,692 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:44,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:44,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206544977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:44,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206544980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:44,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206544980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:44,984 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:44,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:44,986 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206544990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:44,994 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206545000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:45,005 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:45,068 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ff50398d402645819f953f3fa0180319 2024-12-03T06:14:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e1e2d8b9b6844984b6f95ee80446d181 is 50, key is test_row_0/B:col10/1733206482823/Put/seqid=0 2024-12-03T06:14:45,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742234_1410 (size=9757) 2024-12-03T06:14:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206545091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,098 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e1e2d8b9b6844984b6f95ee80446d181 2024-12-03T06:14:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/d0012b35d826482fa17c30cad37d85fe is 50, key is test_row_0/C:col10/1733206482823/Put/seqid=0 2024-12-03T06:14:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742235_1411 (size=9757) 2024-12-03T06:14:45,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206545299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:45,525 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/d0012b35d826482fa17c30cad37d85fe 2024-12-03T06:14:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ff50398d402645819f953f3fa0180319 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319 2024-12-03T06:14:45,546 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319, entries=100, sequenceid=173, filesize=21.9 K 2024-12-03T06:14:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e1e2d8b9b6844984b6f95ee80446d181 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181 2024-12-03T06:14:45,553 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181, entries=100, sequenceid=173, filesize=9.5 K 2024-12-03T06:14:45,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-03T06:14:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/d0012b35d826482fa17c30cad37d85fe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe 2024-12-03T06:14:45,564 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe, entries=100, sequenceid=173, filesize=9.5 K 2024-12-03T06:14:45,566 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c675065cc3f30321d4b6a3915fdea885 in 1379ms, sequenceid=173, compaction requested=false 2024-12-03T06:14:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-03T06:14:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-03T06:14:45,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T06:14:45,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1460 sec 2024-12-03T06:14:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:45,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.1520 sec 2024-12-03T06:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:45,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:45,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033395da45f1634b5eaca71e250d8b9be8_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:45,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742236_1412 (size=14794) 2024-12-03T06:14:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206545695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:45,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206545820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:46,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206546040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:46,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:46,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033395da45f1634b5eaca71e250d8b9be8_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033395da45f1634b5eaca71e250d8b9be8_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:46,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ebfc47ab95e54942a7c4fb308f8f8150, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ebfc47ab95e54942a7c4fb308f8f8150 is 175, key is test_row_0/A:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742237_1413 (size=39749) 2024-12-03T06:14:46,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ebfc47ab95e54942a7c4fb308f8f8150 2024-12-03T06:14:46,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/1843eca712fa4448a6738df9a9ba1d25 is 50, key is test_row_0/B:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742238_1414 (size=12151) 2024-12-03T06:14:46,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/1843eca712fa4448a6738df9a9ba1d25 2024-12-03T06:14:46,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/806aa72fd6984762a2048917317b9ed4 is 50, key is test_row_0/C:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742239_1415 (size=12151) 2024-12-03T06:14:46,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/806aa72fd6984762a2048917317b9ed4 2024-12-03T06:14:46,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ebfc47ab95e54942a7c4fb308f8f8150 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150 2024-12-03T06:14:46,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150, entries=200, sequenceid=197, filesize=38.8 K 2024-12-03T06:14:46,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/1843eca712fa4448a6738df9a9ba1d25 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25 2024-12-03T06:14:46,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25, entries=150, sequenceid=197, filesize=11.9 K 2024-12-03T06:14:46,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/806aa72fd6984762a2048917317b9ed4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4 2024-12-03T06:14:46,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4, entries=150, sequenceid=197, filesize=11.9 K 2024-12-03T06:14:46,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c675065cc3f30321d4b6a3915fdea885 in 581ms, sequenceid=197, compaction requested=true 2024-12-03T06:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:46,196 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:46,196 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:46,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:46,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:46,200 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:46,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:46,200 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:46,200 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:46,200 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:46,200 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:46,200 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0de03a21a6d748e2baadd7e99b0f1de5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=33.7 K 2024-12-03T06:14:46,200 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=91.5 K 2024-12-03T06:14:46,201 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150] 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de03a21a6d748e2baadd7e99b0f1de5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480634 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c772fd8040ee40ec9da13055ea27db72, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480634 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e2d8b9b6844984b6f95ee80446d181, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206482822 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff50398d402645819f953f3fa0180319, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206482822 2024-12-03T06:14:46,201 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1843eca712fa4448a6738df9a9ba1d25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484961 2024-12-03T06:14:46,202 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfc47ab95e54942a7c4fb308f8f8150, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484950 2024-12-03T06:14:46,212 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:46,212 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b5ce629e35694ad1ba865fdd65956eab is 50, key is test_row_0/B:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,213 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,224 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120395d5d7d396d54f3fad47361f542b1745_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,225 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120395d5d7d396d54f3fad47361f542b1745_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,225 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395d5d7d396d54f3fad47361f542b1745_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742240_1416 (size=12663) 2024-12-03T06:14:46,243 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b5ce629e35694ad1ba865fdd65956eab as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5ce629e35694ad1ba865fdd65956eab 2024-12-03T06:14:46,251 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into b5ce629e35694ad1ba865fdd65956eab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:46,251 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:46,251 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206486196; duration=0sec 2024-12-03T06:14:46,251 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:46,251 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:46,251 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:46,252 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:46,252 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:46,252 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:46,252 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/959b28c6c9224445bc7ee5fc4c5847f4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=33.7 K 2024-12-03T06:14:46,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 959b28c6c9224445bc7ee5fc4c5847f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733206480634 2024-12-03T06:14:46,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d0012b35d826482fa17c30cad37d85fe, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733206482822 2024-12-03T06:14:46,253 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 806aa72fd6984762a2048917317b9ed4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484961 2024-12-03T06:14:46,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742241_1417 (size=4469) 2024-12-03T06:14:46,267 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:46,268 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/3027e1786241424b92f4fcbbb9d45a35 is 50, key is test_row_0/C:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742242_1418 (size=12663) 2024-12-03T06:14:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:46,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:46,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038087fd8aae0f48a3a08063ce0823f12b_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206486361/Put/seqid=0 2024-12-03T06:14:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742243_1419 (size=14794) 2024-12-03T06:14:46,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:46,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038087fd8aae0f48a3a08063ce0823f12b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038087fd8aae0f48a3a08063ce0823f12b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:46,434 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/dc789cda69f14aa0b0ff51d5d5a871b6, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:46,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/dc789cda69f14aa0b0ff51d5d5a871b6 is 175, key is test_row_0/A:col10/1733206486361/Put/seqid=0 2024-12-03T06:14:46,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742244_1420 (size=39749) 2024-12-03T06:14:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206546540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:46,661 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#352 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:46,661 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ab188bc25dbe4586a6a9a586df6ff0ae is 175, key is test_row_0/A:col10/1733206484974/Put/seqid=0 2024-12-03T06:14:46,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742245_1421 (size=31617) 2024-12-03T06:14:46,669 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/ab188bc25dbe4586a6a9a586df6ff0ae as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae 2024-12-03T06:14:46,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206546659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:46,673 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into ab188bc25dbe4586a6a9a586df6ff0ae(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:46,673 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:46,674 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206486196; duration=0sec 2024-12-03T06:14:46,674 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:46,674 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:46,704 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/3027e1786241424b92f4fcbbb9d45a35 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/3027e1786241424b92f4fcbbb9d45a35 2024-12-03T06:14:46,710 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 3027e1786241424b92f4fcbbb9d45a35(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:46,710 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:46,710 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206486196; duration=0sec 2024-12-03T06:14:46,710 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:46,710 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:46,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/dc789cda69f14aa0b0ff51d5d5a871b6 2024-12-03T06:14:46,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/c56fb25164484b3d9e449c9be3d43753 is 50, key is test_row_0/B:col10/1733206486361/Put/seqid=0 2024-12-03T06:14:46,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742246_1422 (size=12151) 2024-12-03T06:14:46,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/c56fb25164484b3d9e449c9be3d43753 2024-12-03T06:14:46,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/716c9d756ab8432a9ace9e1580ab9707 is 50, key is test_row_0/C:col10/1733206486361/Put/seqid=0 2024-12-03T06:14:46,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742247_1423 (size=12151) 2024-12-03T06:14:46,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206546879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206546999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:47,002 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206547009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206547009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:47,015 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:47,015 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206547020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:47,024 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206547189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:47,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/716c9d756ab8432a9ace9e1580ab9707 2024-12-03T06:14:47,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/dc789cda69f14aa0b0ff51d5d5a871b6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6 2024-12-03T06:14:47,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6, entries=200, sequenceid=210, filesize=38.8 K 2024-12-03T06:14:47,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/c56fb25164484b3d9e449c9be3d43753 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753 2024-12-03T06:14:47,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T06:14:47,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/716c9d756ab8432a9ace9e1580ab9707 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707 2024-12-03T06:14:47,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T06:14:47,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c675065cc3f30321d4b6a3915fdea885 in 947ms, sequenceid=210, compaction requested=false 2024-12-03T06:14:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T06:14:47,590 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:47,590 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T06:14:47,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:47,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:47,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206486457, max timestamp=1733206487702Num uniques 859; 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206486457, max timestamp=1733206487702Num uniques 859; 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206486457, max timestamp=1733206487702Num uniques 859; 2024-12-03T06:14:47,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:47,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:47,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035ff85a8543024474868d73f2aa04aa1a_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742248_1424 (size=14794) 2024-12-03T06:14:47,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:47,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206487773, max timestamp=1733206487806Num uniques 859; 2024-12-03T06:14:47,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206487773, max timestamp=1733206487806Num uniques 859; 2024-12-03T06:14:47,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206487773, max timestamp=1733206487806Num uniques 859; 2024-12-03T06:14:47,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:47,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:47,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206487878, max timestamp=1733206487916Num uniques 1018; 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206487878, max timestamp=1733206487916Num uniques 1018; 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733206487878, max timestamp=1733206487916Num uniques 1018; 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:14:47,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206487971, max timestamp=1733206488015Num uniques 1027; 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206487971, max timestamp=1733206488015Num uniques 1027; 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733206487971, max timestamp=1733206488015Num uniques 1027; 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:14:48,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T06:14:48,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733206488046, max timestamp=1733206488116Num uniques 1036; 2024-12-03T06:14:48,119 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:48,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T06:14:48,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733206488046, max timestamp=1733206488116Num uniques 1036; 2024-12-03T06:14:48,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T06:14:48,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733206488046, max timestamp=1733206488116Num uniques 1036; 2024-12-03T06:14:48,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T06:14:48,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035ff85a8543024474868d73f2aa04aa1a_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ff85a8543024474868d73f2aa04aa1a_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:48,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/58aa6c36a81f425dba3d0c7733fb68d6, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:48,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/58aa6c36a81f425dba3d0c7733fb68d6 is 175, key is test_row_0/A:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:48,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742249_1425 (size=39749) 2024-12-03T06:14:48,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/58aa6c36a81f425dba3d0c7733fb68d6 2024-12-03T06:14:48,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e90a6648f2e94510aec620bac678884b is 50, key is test_row_0/B:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:48,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742250_1426 (size=12151) 2024-12-03T06:14:48,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733206488206, max timestamp=1733206488236Num uniques 1045; 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733206488206, max timestamp=1733206488236Num uniques 1045; 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733206488206, max timestamp=1733206488236Num uniques 1045; 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T06:14:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206488326, max timestamp=1733206488344Num uniques 1104; 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206488326, max timestamp=1733206488344Num uniques 1104; 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733206488326, max timestamp=1733206488344Num uniques 1104; 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T06:14:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T06:14:48,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733206488407, max timestamp=1733206488460Num uniques 1013; 2024-12-03T06:14:48,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733206488407, max timestamp=1733206488460Num uniques 1013; 2024-12-03T06:14:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733206488407, max timestamp=1733206488460Num uniques 1013; 2024-12-03T06:14:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T06:14:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T06:14:48,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T06:14:48,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e90a6648f2e94510aec620bac678884b 2024-12-03T06:14:48,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/1db8dd1f7c424361a9547113ece849bf is 50, key is test_row_0/C:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742251_1427 (size=12151) 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733206488559, max timestamp=1733206488611Num uniques 1022; 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733206488559, max timestamp=1733206488611Num uniques 1022; 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733206488559, max timestamp=1733206488611Num uniques 1022; 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-03T06:14:48,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-03T06:14:48,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733206488698, max timestamp=1733206488736Num uniques 1031; 2024-12-03T06:14:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T06:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733206488698, max timestamp=1733206488736Num uniques 1031; 2024-12-03T06:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733206488698, max timestamp=1733206488736Num uniques 1031; 2024-12-03T06:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T06:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T06:14:48,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733206488805, max timestamp=1733206488882Num uniques 1090; 2024-12-03T06:14:48,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-03T06:14:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733206488805, max timestamp=1733206488882Num uniques 1090; 2024-12-03T06:14:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-03T06:14:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733206488805, max timestamp=1733206488882Num uniques 1090; 2024-12-03T06:14:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-03T06:14:48,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:48,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/1db8dd1f7c424361a9547113ece849bf 2024-12-03T06:14:48,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/58aa6c36a81f425dba3d0c7733fb68d6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6 2024-12-03T06:14:48,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6, entries=200, sequenceid=233, filesize=38.8 K 2024-12-03T06:14:48,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/e90a6648f2e94510aec620bac678884b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b 2024-12-03T06:14:48,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b, entries=150, sequenceid=233, filesize=11.9 K 2024-12-03T06:14:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/1db8dd1f7c424361a9547113ece849bf as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf 2024-12-03T06:14:48,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf, entries=150, sequenceid=233, filesize=11.9 K 2024-12-03T06:14:48,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=127.47 KB/130530 for c675065cc3f30321d4b6a3915fdea885 in 1289ms, sequenceid=233, compaction requested=true 2024-12-03T06:14:48,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:48,995 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:48,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:48,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:48,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:48,995 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:48,996 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,996 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,996 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5ce629e35694ad1ba865fdd65956eab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=36.1 K 2024-12-03T06:14:48,996 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=108.5 K 2024-12-03T06:14:48,996 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6] 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ce629e35694ad1ba865fdd65956eab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484961 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab188bc25dbe4586a6a9a586df6ff0ae, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484961 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c56fb25164484b3d9e449c9be3d43753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733206485651 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc789cda69f14aa0b0ff51d5d5a871b6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733206485651 2024-12-03T06:14:48,996 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e90a6648f2e94510aec620bac678884b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:48,997 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58aa6c36a81f425dba3d0c7733fb68d6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:49,001 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,001 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:49,002 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/2f7107e44600485d8f9e4518f98365ef is 50, key is test_row_0/B:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:49,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=134.18 KB heapSize=377.39 KB 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:49,003 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c2dab2bfcd1a4a00ace963086eb0857a_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:14:49,004 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c2dab2bfcd1a4a00ace963086eb0857a_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,004 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2dab2bfcd1a4a00ace963086eb0857a_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742252_1428 (size=12765) 2024-12-03T06:14:49,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dce486c37a1d42c1b35cefad18a13c51_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206488971/Put/seqid=0 2024-12-03T06:14:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742253_1429 (size=4469) 2024-12-03T06:14:49,019 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#361 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:49,019 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/3b575762f747416ea10fec032672f39b is 175, key is test_row_0/A:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:49,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742254_1430 (size=14994) 2024-12-03T06:14:49,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742255_1431 (size=31719) 2024-12-03T06:14:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206549064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:49,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206549180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:49,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:49,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206549399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:49,412 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/2f7107e44600485d8f9e4518f98365ef as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/2f7107e44600485d8f9e4518f98365ef 2024-12-03T06:14:49,415 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into 2f7107e44600485d8f9e4518f98365ef(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:49,415 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:49,415 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=13, startTime=1733206488995; duration=0sec 2024-12-03T06:14:49,416 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:49,416 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:49,416 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:14:49,416 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:14:49,416 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:49,416 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,416 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/3027e1786241424b92f4fcbbb9d45a35, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=36.1 K 2024-12-03T06:14:49,417 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3027e1786241424b92f4fcbbb9d45a35, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733206484961 2024-12-03T06:14:49,417 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 716c9d756ab8432a9ace9e1580ab9707, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733206485651 2024-12-03T06:14:49,417 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db8dd1f7c424361a9547113ece849bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:49,423 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:49,423 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/675a61f8cca6423289127559271f72b1 is 50, key is test_row_0/C:col10/1733206487702/Put/seqid=0 2024-12-03T06:14:49,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:49,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742256_1432 (size=12765) 2024-12-03T06:14:49,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dce486c37a1d42c1b35cefad18a13c51_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dce486c37a1d42c1b35cefad18a13c51_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:49,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e6f9295bcfca4f27a573dd311b36e1f8, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e6f9295bcfca4f27a573dd311b36e1f8 is 175, key is test_row_0/A:col10/1733206488971/Put/seqid=0 2024-12-03T06:14:49,432 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/3b575762f747416ea10fec032672f39b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b 2024-12-03T06:14:49,433 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/675a61f8cca6423289127559271f72b1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/675a61f8cca6423289127559271f72b1 2024-12-03T06:14:49,436 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into 3b575762f747416ea10fec032672f39b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:49,436 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:49,437 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=13, startTime=1733206488994; duration=0sec 2024-12-03T06:14:49,437 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:49,437 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:49,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 675a61f8cca6423289127559271f72b1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:49,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:49,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=13, startTime=1733206488995; duration=0sec 2024-12-03T06:14:49,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:49,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:49,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742257_1433 (size=39949) 2024-12-03T06:14:49,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e6f9295bcfca4f27a573dd311b36e1f8 2024-12-03T06:14:49,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/0f14767f7c45443e9095c45ae65406f4 is 50, key is test_row_0/B:col10/1733206488971/Put/seqid=0 2024-12-03T06:14:49,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742258_1434 (size=12301) 2024-12-03T06:14:49,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/0f14767f7c45443e9095c45ae65406f4 2024-12-03T06:14:49,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/967bd833c09f447f9621dadef40e383e is 50, key is test_row_0/C:col10/1733206488971/Put/seqid=0 2024-12-03T06:14:49,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742259_1435 (size=12301) 2024-12-03T06:14:49,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206549709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:49,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:49,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/967bd833c09f447f9621dadef40e383e 2024-12-03T06:14:49,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/e6f9295bcfca4f27a573dd311b36e1f8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8 2024-12-03T06:14:49,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8, entries=200, sequenceid=423, filesize=39.0 K 2024-12-03T06:14:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/0f14767f7c45443e9095c45ae65406f4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4 2024-12-03T06:14:49,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4, entries=150, sequenceid=423, filesize=12.0 K 2024-12-03T06:14:49,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/967bd833c09f447f9621dadef40e383e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e 2024-12-03T06:14:49,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e, entries=150, sequenceid=423, filesize=12.0 K 2024-12-03T06:14:49,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~377.34 KB/386400, currentSize=46.96 KB/48090 for c675065cc3f30321d4b6a3915fdea885 in 876ms, sequenceid=423, compaction requested=false 2024-12-03T06:14:49,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:49,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T06:14:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:49,882 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:49,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036f31860bf16c4d18916630ad73b0d6ad_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206489045/Put/seqid=0 2024-12-03T06:14:49,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742260_1436 (size=12454) 2024-12-03T06:14:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:49,896 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036f31860bf16c4d18916630ad73b0d6ad_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f31860bf16c4d18916630ad73b0d6ad_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/078a21cb0085454f8a50ba2bc30c42e3, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:49,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/078a21cb0085454f8a50ba2bc30c42e3 is 175, key is test_row_0/A:col10/1733206489045/Put/seqid=0 2024-12-03T06:14:49,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742261_1437 (size=31255) 2024-12-03T06:14:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:50,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:50,303 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/078a21cb0085454f8a50ba2bc30c42e3 2024-12-03T06:14:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/47abe6cef416421fa08e8db4263da019 is 50, key is test_row_0/B:col10/1733206489045/Put/seqid=0 2024-12-03T06:14:50,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206490286, max timestamp=1733206490308Num uniques 859; 2024-12-03T06:14:50,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206490286, max timestamp=1733206490308Num uniques 859; 2024-12-03T06:14:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206490286, max timestamp=1733206490308Num uniques 859; 2024-12-03T06:14:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:14:50,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742262_1438 (size=12301) 2024-12-03T06:14:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206550428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206550540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:50,714 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/47abe6cef416421fa08e8db4263da019 2024-12-03T06:14:50,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/a42cb5e38600470d889547bfddc735ad is 50, key is test_row_0/C:col10/1733206489045/Put/seqid=0 2024-12-03T06:14:50,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742263_1439 (size=12301) 2024-12-03T06:14:50,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206550749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35186 deadline: 1733206551030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733206551030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,039 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,039 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1733206551059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35194 deadline: 1733206551059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,067 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206551059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,068 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in c675065cc3f30321d4b6a3915fdea885 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,126 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/a42cb5e38600470d889547bfddc735ad 2024-12-03T06:14:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/078a21cb0085454f8a50ba2bc30c42e3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3 2024-12-03T06:14:51,132 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3, entries=150, sequenceid=436, filesize=30.5 K 2024-12-03T06:14:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/47abe6cef416421fa08e8db4263da019 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019 2024-12-03T06:14:51,135 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019, entries=150, sequenceid=436, filesize=12.0 K 2024-12-03T06:14:51,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/a42cb5e38600470d889547bfddc735ad as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad 2024-12-03T06:14:51,138 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad, entries=150, sequenceid=436, filesize=12.0 K 2024-12-03T06:14:51,139 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c675065cc3f30321d4b6a3915fdea885 in 1257ms, sequenceid=436, compaction requested=true 2024-12-03T06:14:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T06:14:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T06:14:51,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T06:14:51,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5480 sec 2024-12-03T06:14:51,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.5520 sec 2024-12-03T06:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:51,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T06:14:51,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038e7cdcc3c5664a289543c09201f7dda7_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:51,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742264_1440 (size=14994) 2024-12-03T06:14:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206551655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T06:14:51,760 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:51,761 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T06:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:51,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:14:51,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:14:51,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:14:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206551779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:51,789 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-12-03T06:14:51,789 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:51,789 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:51,789 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-12-03T06:14:51,789 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:51,789 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-12-03T06:14:51,789 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:51,789 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:51,789 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:51,798 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-12-03T06:14:51,798 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-12-03T06:14:51,798 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:51,798 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:51,798 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,798 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,799 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,799 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:51,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:51,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:51,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:51,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206552000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:52,005 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:52,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038e7cdcc3c5664a289543c09201f7dda7_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e7cdcc3c5664a289543c09201f7dda7_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:52,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a8a096754bd24efdb19c97bc78d7da25, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:52,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a8a096754bd24efdb19c97bc78d7da25 is 175, key is test_row_0/A:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:52,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742265_1441 (size=39949) 2024-12-03T06:14:52,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:52,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206552310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:52,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:52,416 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a8a096754bd24efdb19c97bc78d7da25 2024-12-03T06:14:52,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b07ec75303c0402b9c1e446fb6a91d39 is 50, key is test_row_0/B:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742266_1442 (size=12301) 2024-12-03T06:14:52,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1733206552820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:14:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:14:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:14:52,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b07ec75303c0402b9c1e446fb6a91d39 2024-12-03T06:14:52,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:52,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:52,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/30cf29955c6343fa9246a912a0177ca0 is 50, key is test_row_0/C:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:52,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742267_1443 (size=12301) 2024-12-03T06:14:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:53,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:53,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:53,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:53,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:14:53,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/30cf29955c6343fa9246a912a0177ca0 2024-12-03T06:14:53,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/a8a096754bd24efdb19c97bc78d7da25 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25 2024-12-03T06:14:53,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25, entries=200, sequenceid=474, filesize=39.0 K 2024-12-03T06:14:53,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/b07ec75303c0402b9c1e446fb6a91d39 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39 2024-12-03T06:14:53,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39, entries=150, sequenceid=474, filesize=12.0 K 2024-12-03T06:14:53,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/30cf29955c6343fa9246a912a0177ca0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0 2024-12-03T06:14:53,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0, entries=150, sequenceid=474, filesize=12.0 K 2024-12-03T06:14:53,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for c675065cc3f30321d4b6a3915fdea885 in 1709ms, sequenceid=474, compaction requested=true 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:14:53,297 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c675065cc3f30321d4b6a3915fdea885:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:14:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:53,297 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/B is initiating minor compaction (all files) 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/A is initiating minor compaction (all files) 2024-12-03T06:14:53,298 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/B in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,298 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/A in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,298 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/2f7107e44600485d8f9e4518f98365ef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=48.5 K 2024-12-03T06:14:53,298 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=139.5 K 2024-12-03T06:14:53,298 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25] 2024-12-03T06:14:53,298 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7107e44600485d8f9e4518f98365ef, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b575762f747416ea10fec032672f39b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f14767f7c45443e9095c45ae65406f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733206488971 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f9295bcfca4f27a573dd311b36e1f8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733206488971 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 47abe6cef416421fa08e8db4263da019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733206489038 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078a21cb0085454f8a50ba2bc30c42e3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733206489038 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b07ec75303c0402b9c1e446fb6a91d39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733206490400 2024-12-03T06:14:53,299 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8a096754bd24efdb19c97bc78d7da25, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733206490400 2024-12-03T06:14:53,305 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:53,305 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/fd88c12f8dd944859e42b878958a349a is 50, key is test_row_0/B:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:53,307 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:53,309 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412039adf4aa50c884eafae84c7073a87ef87_c675065cc3f30321d4b6a3915fdea885 store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742268_1444 (size=13051) 2024-12-03T06:14:53,325 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412039adf4aa50c884eafae84c7073a87ef87_c675065cc3f30321d4b6a3915fdea885, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:53,326 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039adf4aa50c884eafae84c7073a87ef87_c675065cc3f30321d4b6a3915fdea885 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:53,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742269_1445 (size=4469) 2024-12-03T06:14:53,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,335 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:14:53,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:14:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cccbde0f9ca740c98645dfb58c5ac01e_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_0/A:col10/1733206491630/Put/seqid=0 2024-12-03T06:14:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742270_1446 (size=12454) 2024-12-03T06:14:53,717 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/fd88c12f8dd944859e42b878958a349a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/fd88c12f8dd944859e42b878958a349a 2024-12-03T06:14:53,724 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/B of c675065cc3f30321d4b6a3915fdea885 into fd88c12f8dd944859e42b878958a349a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:53,724 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:53,724 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/B, priority=12, startTime=1733206493297; duration=0sec 2024-12-03T06:14:53,724 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:14:53,724 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:B 2024-12-03T06:14:53,724 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:14:53,726 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:14:53,726 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): c675065cc3f30321d4b6a3915fdea885/C is initiating minor compaction (all files) 2024-12-03T06:14:53,726 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c675065cc3f30321d4b6a3915fdea885/C in TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:53,726 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/675a61f8cca6423289127559271f72b1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp, totalSize=48.5 K 2024-12-03T06:14:53,726 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 675a61f8cca6423289127559271f72b1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733206486457 2024-12-03T06:14:53,727 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 967bd833c09f447f9621dadef40e383e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733206488971 2024-12-03T06:14:53,727 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a42cb5e38600470d889547bfddc735ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733206489038 2024-12-03T06:14:53,728 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 30cf29955c6343fa9246a912a0177ca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733206490400 2024-12-03T06:14:53,730 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#A#compaction#373 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:53,731 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/c64a5777aff744b183f0fb09c711aa57 is 175, key is test_row_0/A:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:53,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742271_1447 (size=32005) 2024-12-03T06:14:53,740 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/c64a5777aff744b183f0fb09c711aa57 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c64a5777aff744b183f0fb09c711aa57 2024-12-03T06:14:53,740 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c675065cc3f30321d4b6a3915fdea885#C#compaction#375 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:14:53,741 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/5b52a0505ca44e73a8b3ba997f73823b is 50, key is test_row_0/C:col10/1733206490400/Put/seqid=0 2024-12-03T06:14:53,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742272_1448 (size=13051) 2024-12-03T06:14:53,745 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/A of c675065cc3f30321d4b6a3915fdea885 into c64a5777aff744b183f0fb09c711aa57(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:53,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:14:53,745 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:53,745 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/A, priority=12, startTime=1733206493297; duration=0sec 2024-12-03T06:14:53,745 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:53,745 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:A 2024-12-03T06:14:53,748 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cccbde0f9ca740c98645dfb58c5ac01e_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cccbde0f9ca740c98645dfb58c5ac01e_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/62da81b1dfd64337b58b04e8f90b87c6, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:14:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/62da81b1dfd64337b58b04e8f90b87c6 is 175, key is test_row_0/A:col10/1733206491630/Put/seqid=0 2024-12-03T06:14:53,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742273_1449 (size=31255) 2024-12-03T06:14:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:14:53,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. as already flushing 2024-12-03T06:14:53,831 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-12-03T06:14:53,831 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:14:53,831 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:53,831 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:14:53,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:14:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:54,153 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/62da81b1dfd64337b58b04e8f90b87c6 2024-12-03T06:14:54,154 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/5b52a0505ca44e73a8b3ba997f73823b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/5b52a0505ca44e73a8b3ba997f73823b 2024-12-03T06:14:54,159 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c675065cc3f30321d4b6a3915fdea885/C of c675065cc3f30321d4b6a3915fdea885 into 5b52a0505ca44e73a8b3ba997f73823b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:14:54,159 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:54,159 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., storeName=c675065cc3f30321d4b6a3915fdea885/C, priority=12, startTime=1733206493297; duration=0sec 2024-12-03T06:14:54,159 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:14:54,159 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c675065cc3f30321d4b6a3915fdea885:C 2024-12-03T06:14:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/05bc87ad999c496ebae68863a61cbcde is 50, key is test_row_0/B:col10/1733206491630/Put/seqid=0 2024-12-03T06:14:54,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742274_1450 (size=12301) 2024-12-03T06:14:54,567 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/05bc87ad999c496ebae68863a61cbcde 2024-12-03T06:14:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/ca21228275e3485d8777153475450571 is 50, key is test_row_0/C:col10/1733206491630/Put/seqid=0 2024-12-03T06:14:54,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742275_1451 (size=12301) 2024-12-03T06:14:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/ca21228275e3485d8777153475450571 2024-12-03T06:14:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/62da81b1dfd64337b58b04e8f90b87c6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/62da81b1dfd64337b58b04e8f90b87c6 2024-12-03T06:14:55,001 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/62da81b1dfd64337b58b04e8f90b87c6, entries=150, sequenceid=483, filesize=30.5 K 2024-12-03T06:14:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/05bc87ad999c496ebae68863a61cbcde as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/05bc87ad999c496ebae68863a61cbcde 2024-12-03T06:14:55,007 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/05bc87ad999c496ebae68863a61cbcde, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T06:14:55,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/ca21228275e3485d8777153475450571 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/ca21228275e3485d8777153475450571 2024-12-03T06:14:55,011 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/ca21228275e3485d8777153475450571, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T06:14:55,012 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for c675065cc3f30321d4b6a3915fdea885 in 1677ms, sequenceid=483, compaction requested=false 2024-12-03T06:14:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c675065cc3f30321d4b6a3915fdea885: 2024-12-03T06:14:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:14:55,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T06:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T06:14:55,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T06:14:55,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2470 sec 2024-12-03T06:14:55,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 3.2560 sec 2024-12-03T06:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T06:14:55,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:14:59,769 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:15:01,082 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-12-03T06:15:01,082 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:01,082 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,082 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:01,111 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-12-03T06:15:01,111 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-12-03T06:15:01,111 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:01,111 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:01,112 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,112 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,112 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,112 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:01,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:01,171 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-12-03T06:15:01,171 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:01,171 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,171 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 309 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1466 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4396 rows 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1469 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4397 rows 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1460 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4379 rows 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1470 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4398 rows 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1477 2024-12-03T06:15:01,171 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4422 rows 2024-12-03T06:15:01,172 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:15:01,172 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:15:01,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:01,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:01,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:01,175 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206501175"}]},"ts":"1733206501175"} 2024-12-03T06:15:01,177 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:15:01,177 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:15:01,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:15:01,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, UNASSIGN}] 2024-12-03T06:15:01,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, UNASSIGN 2024-12-03T06:15:01,180 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:01,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, UNASSIGN because future has completed 2024-12-03T06:15:01,182 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:15:01,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:01,334 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing c675065cc3f30321d4b6a3915fdea885, disabling compactions & flushes 2024-12-03T06:15:01,334 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. after waiting 0 ms 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:15:01,334 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing c675065cc3f30321d4b6a3915fdea885 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=A 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=B 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c675065cc3f30321d4b6a3915fdea885, store=C 2024-12-03T06:15:01,334 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:01,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ab72334a9cc54643a58a69e7eca9ba9b_c675065cc3f30321d4b6a3915fdea885 is 50, key is test_row_1/A:col10/1733206501169/Put/seqid=0 2024-12-03T06:15:01,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742276_1452 (size=9914) 2024-12-03T06:15:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:01,744 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:01,751 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ab72334a9cc54643a58a69e7eca9ba9b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ab72334a9cc54643a58a69e7eca9ba9b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:01,752 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f, store: [table=TestAcidGuarantees family=A region=c675065cc3f30321d4b6a3915fdea885] 2024-12-03T06:15:01,752 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f is 175, key is test_row_1/A:col10/1733206501169/Put/seqid=0 2024-12-03T06:15:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742277_1453 (size=22561) 2024-12-03T06:15:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:02,159 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f 2024-12-03T06:15:02,164 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/df146c9e8d4540fdb0ff03834cc9c050 is 50, key is test_row_1/B:col10/1733206501169/Put/seqid=0 2024-12-03T06:15:02,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742278_1454 (size=9857) 2024-12-03T06:15:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:02,569 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/df146c9e8d4540fdb0ff03834cc9c050 2024-12-03T06:15:02,578 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/8a3ad440f82e433e85a92c9c7b3fdece is 50, key is test_row_1/C:col10/1733206501169/Put/seqid=0 2024-12-03T06:15:02,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742279_1455 (size=9857) 2024-12-03T06:15:02,982 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/8a3ad440f82e433e85a92c9c7b3fdece 2024-12-03T06:15:02,985 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f 2024-12-03T06:15:02,988 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f, entries=100, sequenceid=494, filesize=22.0 K 2024-12-03T06:15:02,989 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/B/df146c9e8d4540fdb0ff03834cc9c050 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/df146c9e8d4540fdb0ff03834cc9c050 2024-12-03T06:15:02,992 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/df146c9e8d4540fdb0ff03834cc9c050, entries=100, sequenceid=494, filesize=9.6 K 2024-12-03T06:15:02,993 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/.tmp/C/8a3ad440f82e433e85a92c9c7b3fdece as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/8a3ad440f82e433e85a92c9c7b3fdece 2024-12-03T06:15:02,996 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/8a3ad440f82e433e85a92c9c7b3fdece, entries=100, sequenceid=494, filesize=9.6 K 2024-12-03T06:15:02,997 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c675065cc3f30321d4b6a3915fdea885 in 1663ms, sequenceid=494, compaction requested=true 2024-12-03T06:15:02,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25] to archive 2024-12-03T06:15:02,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:03,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0701c474c7cc485dafe7c92e6979b13b 2024-12-03T06:15:03,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/13da072b324f47ca9b2eab6b51d2e079 2024-12-03T06:15:03,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/871e7c72ed3444fca4f9fd35338b2f54 2024-12-03T06:15:03,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3a4379010f6443abbfb18eb1430cc114 2024-12-03T06:15:03,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a137ba6adac74880a72778cc5d9620e0 2024-12-03T06:15:03,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e28da75366d34c19815720e4c5e5972e 2024-12-03T06:15:03,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/72caaef430184a6cba36d915667d33d7 2024-12-03T06:15:03,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/5259304f8166414584c1c7c80bd24998 2024-12-03T06:15:03,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e95caf6c5fa042e6b4b606b1a7905cd3 2024-12-03T06:15:03,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/1dcd063a526744d39cee5be9f8bebb5d 2024-12-03T06:15:03,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/783c5e55be284277961857986d3d4909 2024-12-03T06:15:03,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/bcd525c4c3ba4b2ebb0444036b0c4f7b 2024-12-03T06:15:03,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c772fd8040ee40ec9da13055ea27db72 2024-12-03T06:15:03,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ff50398d402645819f953f3fa0180319 2024-12-03T06:15:03,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ebfc47ab95e54942a7c4fb308f8f8150 2024-12-03T06:15:03,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/ab188bc25dbe4586a6a9a586df6ff0ae 2024-12-03T06:15:03,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/dc789cda69f14aa0b0ff51d5d5a871b6 2024-12-03T06:15:03,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/58aa6c36a81f425dba3d0c7733fb68d6 2024-12-03T06:15:03,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/3b575762f747416ea10fec032672f39b 2024-12-03T06:15:03,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/e6f9295bcfca4f27a573dd311b36e1f8 2024-12-03T06:15:03,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/078a21cb0085454f8a50ba2bc30c42e3 2024-12-03T06:15:03,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/a8a096754bd24efdb19c97bc78d7da25 2024-12-03T06:15:03,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/de4eb0b735dd4ad58fc05c66197a4f5d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/4793de6e63c44fd59c65e13414d37f1e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/49190b2f47394b71bffe15affe490860, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0de03a21a6d748e2baadd7e99b0f1de5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5ce629e35694ad1ba865fdd65956eab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/2f7107e44600485d8f9e4518f98365ef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39] to archive 2024-12-03T06:15:03,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:03,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/bea77aa527e143d184be7ef13826cadd 2024-12-03T06:15:03,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/70c29f706e304937a589b9625c98268a 2024-12-03T06:15:03,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/de4eb0b735dd4ad58fc05c66197a4f5d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/de4eb0b735dd4ad58fc05c66197a4f5d 2024-12-03T06:15:03,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/12d446c1ceb04816a49059b033edd006 2024-12-03T06:15:03,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5efcf3f4f0b497b8fd0f82891715248 2024-12-03T06:15:03,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/4793de6e63c44fd59c65e13414d37f1e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/4793de6e63c44fd59c65e13414d37f1e 2024-12-03T06:15:03,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/973bd903b30a4734920393df3af26fac 2024-12-03T06:15:03,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/f267968d3d1346b7bb9874262cbf1b0d 2024-12-03T06:15:03,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/49190b2f47394b71bffe15affe490860 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/49190b2f47394b71bffe15affe490860 2024-12-03T06:15:03,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/57d324dba4f74143b3eec5fa95633067 2024-12-03T06:15:03,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/a97117076c834286bd299e3fa2e9a9e3 2024-12-03T06:15:03,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0de03a21a6d748e2baadd7e99b0f1de5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0de03a21a6d748e2baadd7e99b0f1de5 2024-12-03T06:15:03,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/5bf16acf594d4b0ba30d178ed4faf939 2024-12-03T06:15:03,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e1e2d8b9b6844984b6f95ee80446d181 2024-12-03T06:15:03,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5ce629e35694ad1ba865fdd65956eab to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b5ce629e35694ad1ba865fdd65956eab 2024-12-03T06:15:03,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/1843eca712fa4448a6738df9a9ba1d25 2024-12-03T06:15:03,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/c56fb25164484b3d9e449c9be3d43753 2024-12-03T06:15:03,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/2f7107e44600485d8f9e4518f98365ef to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/2f7107e44600485d8f9e4518f98365ef 2024-12-03T06:15:03,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/e90a6648f2e94510aec620bac678884b 2024-12-03T06:15:03,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/0f14767f7c45443e9095c45ae65406f4 2024-12-03T06:15:03,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/47abe6cef416421fa08e8db4263da019 2024-12-03T06:15:03,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/b07ec75303c0402b9c1e446fb6a91d39 2024-12-03T06:15:03,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/91e4539cde8e4cb9afdf201a75849995, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/43d0e8b7cc714dfaa3374e0f06c64ec8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/92e7b4b2efa94d1e9edde951a8b5fa76, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/959b28c6c9224445bc7ee5fc4c5847f4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/3027e1786241424b92f4fcbbb9d45a35, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/675a61f8cca6423289127559271f72b1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0] to archive 2024-12-03T06:15:03,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:03,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/146c3bf1389240958b2ea6d50f14fb04 2024-12-03T06:15:03,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/4f4fb5b5a552447e915c2c0858c3bb57 2024-12-03T06:15:03,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/91e4539cde8e4cb9afdf201a75849995 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/91e4539cde8e4cb9afdf201a75849995 2024-12-03T06:15:03,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/19d721ab3d1b4268a40c44069a45c7f5 2024-12-03T06:15:03,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/79b7d84c02334c4d9a32fd791f0d2d3c 2024-12-03T06:15:03,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/43d0e8b7cc714dfaa3374e0f06c64ec8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/43d0e8b7cc714dfaa3374e0f06c64ec8 2024-12-03T06:15:03,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/59acb46e16074fcdadf09cc545a46aae 2024-12-03T06:15:03,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/25f7258fa3014763b26ec3f00eee741e 2024-12-03T06:15:03,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/92e7b4b2efa94d1e9edde951a8b5fa76 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/92e7b4b2efa94d1e9edde951a8b5fa76 2024-12-03T06:15:03,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/097d8f299a5648e78a53f90d7d12b615 2024-12-03T06:15:03,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/fab8c0b071ce4dbb864607416f7c1d04 2024-12-03T06:15:03,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/959b28c6c9224445bc7ee5fc4c5847f4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/959b28c6c9224445bc7ee5fc4c5847f4 2024-12-03T06:15:03,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/01ad89bbb4534f72a3f37ca76ceff913 2024-12-03T06:15:03,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/d0012b35d826482fa17c30cad37d85fe 2024-12-03T06:15:03,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/3027e1786241424b92f4fcbbb9d45a35 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/3027e1786241424b92f4fcbbb9d45a35 2024-12-03T06:15:03,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/806aa72fd6984762a2048917317b9ed4 2024-12-03T06:15:03,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/716c9d756ab8432a9ace9e1580ab9707 2024-12-03T06:15:03,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/675a61f8cca6423289127559271f72b1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/675a61f8cca6423289127559271f72b1 2024-12-03T06:15:03,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/1db8dd1f7c424361a9547113ece849bf 2024-12-03T06:15:03,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/967bd833c09f447f9621dadef40e383e 2024-12-03T06:15:03,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/a42cb5e38600470d889547bfddc735ad 2024-12-03T06:15:03,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/30cf29955c6343fa9246a912a0177ca0 2024-12-03T06:15:03,067 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=4 2024-12-03T06:15:03,068 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. 2024-12-03T06:15:03,068 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for c675065cc3f30321d4b6a3915fdea885: Waiting for close lock at 1733206501334Running coprocessor pre-close hooks at 1733206501334Disabling compacts and flushes for region at 1733206501334Disabling writes for close at 1733206501334Obtaining lock to block concurrent updates at 1733206501334Preparing flush snapshotting stores in c675065cc3f30321d4b6a3915fdea885 at 1733206501334Finished memstore snapshotting TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733206501334Flushing stores of TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885. at 1733206501335 (+1 ms)Flushing c675065cc3f30321d4b6a3915fdea885/A: creating writer at 1733206501335Flushing mob file c675065cc3f30321d4b6a3915fdea885/A: appending metadata at 1733206501339 (+4 ms)Flushing mob file c675065cc3f30321d4b6a3915fdea885/A: closing flushed file at 1733206501339Flushing c675065cc3f30321d4b6a3915fdea885/A: appending metadata at 1733206501752 (+413 ms)Flushing c675065cc3f30321d4b6a3915fdea885/A: closing flushed file at 1733206501752Flushing c675065cc3f30321d4b6a3915fdea885/B: creating writer at 1733206502162 (+410 ms)Flushing c675065cc3f30321d4b6a3915fdea885/B: appending metadata at 1733206502164 (+2 ms)Flushing c675065cc3f30321d4b6a3915fdea885/B: closing flushed file at 1733206502164Flushing c675065cc3f30321d4b6a3915fdea885/C: creating writer at 1733206502574 (+410 ms)Flushing c675065cc3f30321d4b6a3915fdea885/C: appending metadata at 1733206502577 (+3 ms)Flushing c675065cc3f30321d4b6a3915fdea885/C: closing flushed file at 1733206502577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1349de16: reopening flushed file at 1733206502985 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5230b71: reopening flushed file at 1733206502988 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ff5951: reopening flushed file at 1733206502992 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c675065cc3f30321d4b6a3915fdea885 in 1663ms, sequenceid=494, compaction requested=true at 1733206502997 (+5 ms)Writing region close event to WAL at 1733206503064 (+67 ms)Running coprocessor post-close hooks at 1733206503068 (+4 ms)Closed at 1733206503068 2024-12-03T06:15:03,070 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=c675065cc3f30321d4b6a3915fdea885, regionState=CLOSED 2024-12-03T06:15:03,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:03,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T06:15:03,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure c675065cc3f30321d4b6a3915fdea885, server=122cddc95ec8,35255,1733206381424 in 1.8920 sec 2024-12-03T06:15:03,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-12-03T06:15:03,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c675065cc3f30321d4b6a3915fdea885, UNASSIGN in 1.8980 sec 2024-12-03T06:15:03,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T06:15:03,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9030 sec 2024-12-03T06:15:03,084 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206503083"}]},"ts":"1733206503083"} 2024-12-03T06:15:03,085 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:15:03,085 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:15:03,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9140 sec 2024-12-03T06:15:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T06:15:03,321 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:03,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,327 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T06:15:03,328 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,336 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits] 2024-12-03T06:15:03,341 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/0ed65a5dc5fc4cb880eb46c3a87b9a8f 2024-12-03T06:15:03,343 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/62da81b1dfd64337b58b04e8f90b87c6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/62da81b1dfd64337b58b04e8f90b87c6 2024-12-03T06:15:03,345 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c64a5777aff744b183f0fb09c711aa57 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/A/c64a5777aff744b183f0fb09c711aa57 2024-12-03T06:15:03,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/05bc87ad999c496ebae68863a61cbcde to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/05bc87ad999c496ebae68863a61cbcde 2024-12-03T06:15:03,352 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/df146c9e8d4540fdb0ff03834cc9c050 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/df146c9e8d4540fdb0ff03834cc9c050 2024-12-03T06:15:03,354 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/fd88c12f8dd944859e42b878958a349a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/B/fd88c12f8dd944859e42b878958a349a 2024-12-03T06:15:03,358 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/5b52a0505ca44e73a8b3ba997f73823b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/5b52a0505ca44e73a8b3ba997f73823b 2024-12-03T06:15:03,360 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/8a3ad440f82e433e85a92c9c7b3fdece to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/8a3ad440f82e433e85a92c9c7b3fdece 2024-12-03T06:15:03,362 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/ca21228275e3485d8777153475450571 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/C/ca21228275e3485d8777153475450571 2024-12-03T06:15:03,367 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits/497.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885/recovered.edits/497.seqid 2024-12-03T06:15:03,368 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,368 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:15:03,369 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:15:03,370 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T06:15:03,376 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120310a331382bb849daa42a0f91660cdd03_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120310a331382bb849daa42a0f91660cdd03_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,379 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c4a76888d9f4d3ebd85dfc4c3a3156d_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c4a76888d9f4d3ebd85dfc4c3a3156d_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,381 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033395da45f1634b5eaca71e250d8b9be8_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033395da45f1634b5eaca71e250d8b9be8_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,384 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335317adf895a49cfa6f9ffc35106623b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335317adf895a49cfa6f9ffc35106623b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,386 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ff85a8543024474868d73f2aa04aa1a_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ff85a8543024474868d73f2aa04aa1a_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,387 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f31860bf16c4d18916630ad73b0d6ad_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f31860bf16c4d18916630ad73b0d6ad_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,389 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203704892b72080480f960692179088232b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203704892b72080480f960692179088232b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,390 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037633b12e0e5e47f3a44b21631ff58c60_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037633b12e0e5e47f3a44b21631ff58c60_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,391 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038087fd8aae0f48a3a08063ce0823f12b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038087fd8aae0f48a3a08063ce0823f12b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,392 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e7cdcc3c5664a289543c09201f7dda7_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e7cdcc3c5664a289543c09201f7dda7_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,394 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ab72334a9cc54643a58a69e7eca9ba9b_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ab72334a9cc54643a58a69e7eca9ba9b_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,395 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b3eb7b2f56ba44728f4385b60c77724f_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b3eb7b2f56ba44728f4385b60c77724f_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,396 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cae0cb169be64f1bb1e4cc37efcb5014_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cae0cb169be64f1bb1e4cc37efcb5014_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,397 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cccbde0f9ca740c98645dfb58c5ac01e_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cccbde0f9ca740c98645dfb58c5ac01e_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,398 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd04b54f61fb4c2485925e59b36d863a_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd04b54f61fb4c2485925e59b36d863a_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,399 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dce486c37a1d42c1b35cefad18a13c51_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dce486c37a1d42c1b35cefad18a13c51_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,400 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f46adc6bcded4545a6d94c9b8c9b6aa6_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f46adc6bcded4545a6d94c9b8c9b6aa6_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,402 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f6c96176c44b5ea55976e1dc8e4f0d_c675065cc3f30321d4b6a3915fdea885 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f6c96176c44b5ea55976e1dc8e4f0d_c675065cc3f30321d4b6a3915fdea885 2024-12-03T06:15:03,402 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:15:03,405 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,408 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:15:03,411 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:15:03,412 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,412 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:15:03,412 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206503412"}]},"ts":"9223372036854775807"} 2024-12-03T06:15:03,414 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:15:03,414 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c675065cc3f30321d4b6a3915fdea885, NAME => 'TestAcidGuarantees,,1733206468637.c675065cc3f30321d4b6a3915fdea885.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:15:03,415 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:15:03,415 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206503415"}]},"ts":"9223372036854775807"} 2024-12-03T06:15:03,417 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:15:03,417 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-12-03T06:15:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T06:15:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:15:03,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:03,450 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=12110 (was 11582) - AvailableMemoryMB LEAK? - 2024-12-03T06:15:03,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=12110 2024-12-03T06:15:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:15:03,462 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:03,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:15:03,464 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:03,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-12-03T06:15:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:03,465 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:15:03,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742280_1456 (size=1061) 2024-12-03T06:15:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:03,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:15:03,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742281_1457 (size=53) 2024-12-03T06:15:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:04,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:15:04,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8965fa85a45aa2b83708e0c9d6c60459, disabling compactions & flushes 2024-12-03T06:15:04,282 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. after waiting 0 ms 2024-12-03T06:15:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8965fa85a45aa2b83708e0c9d6c60459: Waiting for close lock at 1733206504282Disabling compacts and flushes for region at 1733206504282Disabling writes for close at 1733206504283 (+1 ms)Writing region close event to WAL at 1733206504283Closed at 1733206504283 2024-12-03T06:15:04,285 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:15:04,285 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206504285"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206504285"}]},"ts":"1733206504285"} 2024-12-03T06:15:04,287 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:15:04,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:15:04,288 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206504288"}]},"ts":"1733206504288"} 2024-12-03T06:15:04,290 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:15:04,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, ASSIGN}] 2024-12-03T06:15:04,291 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, ASSIGN 2024-12-03T06:15:04,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:15:04,443 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=8965fa85a45aa2b83708e0c9d6c60459, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:04,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, ASSIGN because future has completed 2024-12-03T06:15:04,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:04,524 WARN [regionserver/122cddc95ec8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-12-03T06:15:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:04,601 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,602 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,605 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,607 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:04,608 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8965fa85a45aa2b83708e0c9d6c60459 columnFamilyName A 2024-12-03T06:15:04,608 DEBUG [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:04,609 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(327): Store=8965fa85a45aa2b83708e0c9d6c60459/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:04,609 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,611 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:04,611 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8965fa85a45aa2b83708e0c9d6c60459 columnFamilyName B 2024-12-03T06:15:04,611 DEBUG [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:04,612 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(327): Store=8965fa85a45aa2b83708e0c9d6c60459/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:04,612 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,613 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:04,614 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8965fa85a45aa2b83708e0c9d6c60459 columnFamilyName C 2024-12-03T06:15:04,614 DEBUG [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:04,614 INFO [StoreOpener-8965fa85a45aa2b83708e0c9d6c60459-1 {}] regionserver.HStore(327): Store=8965fa85a45aa2b83708e0c9d6c60459/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:04,614 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,614 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,615 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,615 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,616 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,616 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,616 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:15:04,617 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,619 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:15:04,619 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened 8965fa85a45aa2b83708e0c9d6c60459; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60230582, jitterRate=-0.10249438881874084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:15:04,619 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:04,619 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for 8965fa85a45aa2b83708e0c9d6c60459: Running coprocessor pre-open hook at 1733206504603Writing region info on filesystem at 1733206504603Initializing all the Stores at 1733206504605 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206504605Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206504605Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206504605Cleaning up temporary data from old regions at 1733206504616 (+11 ms)Running coprocessor post-open hooks at 1733206504619 (+3 ms)Region opened successfully at 1733206504619 2024-12-03T06:15:04,620 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., pid=128, masterSystemTime=1733206504598 2024-12-03T06:15:04,621 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,622 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:04,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=8965fa85a45aa2b83708e0c9d6c60459, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:04,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:04,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T06:15:04,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 in 179 msec 2024-12-03T06:15:04,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-12-03T06:15:04,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, ASSIGN in 335 msec 2024-12-03T06:15:04,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:15:04,627 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206504627"}]},"ts":"1733206504627"} 2024-12-03T06:15:04,628 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:15:04,629 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:15:04,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-12-03T06:15:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-12-03T06:15:05,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:05,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bd6f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459b76ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,623 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:15:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:15:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43a61524 2024-12-03T06:15:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:15:05,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:15:05,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,631 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777d8127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55caa11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fffb2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d707fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,638 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1328fc5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,640 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146eb47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,641 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e5f663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481cf968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f5150e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e064011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f12873b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,651 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a34287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,653 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b202455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e50720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c220a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6d17d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1414cb04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f699e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:05,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:05,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602955f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:05,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2197234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:05,666 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,666 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,666 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,666 DEBUG [Thread-2048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,666 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:05,666 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,667 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,667 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,667 DEBUG [Thread-2055 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,667 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,667 DEBUG [Thread-2048 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T06:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,668 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,669 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,670 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,671 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:05,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2] 2024-12-03T06:15:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:05,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:05,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/2dad234ec6be4a1d9d8449403952c839 is 50, key is test_row_0/A:col10/1733206505675/Put/seqid=0 2024-12-03T06:15:05,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742282_1458 (size=12001) 2024-12-03T06:15:05,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/2dad234ec6be4a1d9d8449403952c839 2024-12-03T06:15:05,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733206505708, max timestamp=1733206505727Num uniques 859; 2024-12-03T06:15:05,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-03T06:15:05,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733206505708, max timestamp=1733206505727Num uniques 859; 2024-12-03T06:15:05,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-03T06:15:05,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206505715, max timestamp=1733206505727Num uniques 859; 2024-12-03T06:15:05,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:15:05,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 is 50, key is test_row_0/B:col10/1733206505675/Put/seqid=0 2024-12-03T06:15:05,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742283_1459 (size=12001) 2024-12-03T06:15:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206565752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206565753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206565757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206565758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206565763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:05,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:05,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206565870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206565870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206565873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206565870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206565879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:05,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:05,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206566080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206566080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206566080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206566089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206566099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 2024-12-03T06:15:06,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4fe02173b663454fa95c802c648338c6 is 50, key is test_row_0/C:col10/1733206505675/Put/seqid=0 2024-12-03T06:15:06,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742284_1460 (size=12001) 2024-12-03T06:15:06,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206566390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206566390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206566390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206566399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206566411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4fe02173b663454fa95c802c648338c6 2024-12-03T06:15:06,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/2dad234ec6be4a1d9d8449403952c839 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839 2024-12-03T06:15:06,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:15:06,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 2024-12-03T06:15:06,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:15:06,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4fe02173b663454fa95c802c648338c6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6 2024-12-03T06:15:06,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T06:15:06,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 8965fa85a45aa2b83708e0c9d6c60459 in 926ms, sequenceid=12, compaction requested=false 2024-12-03T06:15:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:06,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:06,742 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=129.71 KB heapSize=356.08 KB 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:06,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:06,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:06,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/586baa48a6314eb79d2e70f479c9b482 is 50, key is test_row_0/A:col10/1733206505761/Put/seqid=0 2024-12-03T06:15:06,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742285_1461 (size=12001) 2024-12-03T06:15:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:06,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:06,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206566913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206566914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206566917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206566918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:06,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206566930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206567029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206567030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206567030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206567030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,149 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/586baa48a6314eb79d2e70f479c9b482 2024-12-03T06:15:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b921a38d202542d3b5344035beda5d0a is 50, key is test_row_0/B:col10/1733206505761/Put/seqid=0 2024-12-03T06:15:07,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742286_1462 (size=12001) 2024-12-03T06:15:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206567240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206567240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206567240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206567240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206567549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206567549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206567549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206567549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:07,560 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b921a38d202542d3b5344035beda5d0a 2024-12-03T06:15:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b00a92483230499cbb894ae3216c07f3 is 50, key is test_row_0/C:col10/1733206505761/Put/seqid=0 2024-12-03T06:15:07,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742287_1463 (size=12001) 2024-12-03T06:15:07,573 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b00a92483230499cbb894ae3216c07f3 2024-12-03T06:15:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/586baa48a6314eb79d2e70f479c9b482 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482 2024-12-03T06:15:07,585 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482, entries=150, sequenceid=45, filesize=11.7 K 2024-12-03T06:15:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b921a38d202542d3b5344035beda5d0a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a 2024-12-03T06:15:07,589 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a, entries=150, sequenceid=45, filesize=11.7 K 2024-12-03T06:15:07,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b00a92483230499cbb894ae3216c07f3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3 2024-12-03T06:15:07,601 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3, entries=150, sequenceid=45, filesize=11.7 K 2024-12-03T06:15:07,602 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~356.03 KB/364576, currentSize=53.67 KB/54960 for 8965fa85a45aa2b83708e0c9d6c60459 in 860ms, sequenceid=45, compaction requested=false 2024-12-03T06:15:07,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:07,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:07,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T06:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T06:15:07,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T06:15:07,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-12-03T06:15:07,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9390 sec 2024-12-03T06:15:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T06:15:07,820 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:07,820 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T06:15:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:07,822 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:07,823 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:07,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:07,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:07,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/419e3c100d5f43189f6edbac23129aab is 50, key is test_row_0/A:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:07,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742288_1464 (size=14341) 2024-12-03T06:15:07,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/419e3c100d5f43189f6edbac23129aab 2024-12-03T06:15:07,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/0e0846d713d84878814c62b556a1be7d is 50, key is test_row_0/B:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:07,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T06:15:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:07,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:07,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742289_1465 (size=12001) 2024-12-03T06:15:07,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/0e0846d713d84878814c62b556a1be7d 2024-12-03T06:15:08,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/15c17831a5264c34925e831790524e8b is 50, key is test_row_0/C:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:08,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742290_1466 (size=12001) 2024-12-03T06:15:08,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206568071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206568072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206568072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206568074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206568078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T06:15:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206568189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206568189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206568189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206568189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206568200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T06:15:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:08,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/15c17831a5264c34925e831790524e8b 2024-12-03T06:15:08,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206568410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206568410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206568410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206568410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/419e3c100d5f43189f6edbac23129aab as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab 2024-12-03T06:15:08,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab, entries=200, sequenceid=57, filesize=14.0 K 2024-12-03T06:15:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/0e0846d713d84878814c62b556a1be7d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d 2024-12-03T06:15:08,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T06:15:08,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/15c17831a5264c34925e831790524e8b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b 2024-12-03T06:15:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206568420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T06:15:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:08,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T06:15:08,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8965fa85a45aa2b83708e0c9d6c60459 in 504ms, sequenceid=57, compaction requested=true 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:08,447 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:08,447 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:08,448 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:08,448 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:08,448 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:08,448 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:08,448 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,448 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=37.4 K 2024-12-03T06:15:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:08,450 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,450 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.2 K 2024-12-03T06:15:08,451 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dad234ec6be4a1d9d8449403952c839, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206505675 2024-12-03T06:15:08,451 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d6a9bebeb349f1a5c7a7fb5b3dfb32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206505675 2024-12-03T06:15:08,454 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 586baa48a6314eb79d2e70f479c9b482, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733206505751 2024-12-03T06:15:08,454 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b921a38d202542d3b5344035beda5d0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733206505751 2024-12-03T06:15:08,455 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 419e3c100d5f43189f6edbac23129aab, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506906 2024-12-03T06:15:08,455 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e0846d713d84878814c62b556a1be7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506914 2024-12-03T06:15:08,470 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:08,471 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/045f51503fbc4ae380d230b044b33742 is 50, key is test_row_0/B:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:08,473 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:08,474 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/b90da0e5442f45e9b3347cca1033fbf4 is 50, key is test_row_0/A:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742291_1467 (size=12104) 2024-12-03T06:15:08,510 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/045f51503fbc4ae380d230b044b33742 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/045f51503fbc4ae380d230b044b33742 2024-12-03T06:15:08,514 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 045f51503fbc4ae380d230b044b33742(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:08,514 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:08,514 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206508447; duration=0sec 2024-12-03T06:15:08,514 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:08,514 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:08,514 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:08,515 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:08,515 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:08,516 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,516 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.2 K 2024-12-03T06:15:08,518 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe02173b663454fa95c802c648338c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733206505675 2024-12-03T06:15:08,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742292_1468 (size=12104) 2024-12-03T06:15:08,520 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b00a92483230499cbb894ae3216c07f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733206505751 2024-12-03T06:15:08,520 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c17831a5264c34925e831790524e8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506914 2024-12-03T06:15:08,543 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:08,543 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/527e08d957684094801cf276756c7ec0 is 50, key is test_row_0/C:col10/1733206506916/Put/seqid=0 2024-12-03T06:15:08,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742293_1469 (size=12104) 2024-12-03T06:15:08,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:08,589 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/eae35a5e76b64c1d9ca14e9c5fea0a9e is 50, key is test_row_0/A:col10/1733206508038/Put/seqid=0 2024-12-03T06:15:08,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:15:08,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742294_1470 (size=12001) 2024-12-03T06:15:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:08,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:08,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206568745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206568745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206568750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206568750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206568751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206568860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206568860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206568860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206568869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206568869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:08,925 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/b90da0e5442f45e9b3347cca1033fbf4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/b90da0e5442f45e9b3347cca1033fbf4 2024-12-03T06:15:08,932 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into b90da0e5442f45e9b3347cca1033fbf4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:08,932 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:08,932 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206508447; duration=0sec 2024-12-03T06:15:08,932 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:08,932 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:08,984 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/527e08d957684094801cf276756c7ec0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/527e08d957684094801cf276756c7ec0 2024-12-03T06:15:08,997 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into 527e08d957684094801cf276756c7ec0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:08,997 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:08,997 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206508447; duration=0sec 2024-12-03T06:15:08,997 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:08,997 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:09,047 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/eae35a5e76b64c1d9ca14e9c5fea0a9e 2024-12-03T06:15:09,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/26f1c24b8cee4a9cb3d64783d064dae0 is 50, key is test_row_0/B:col10/1733206508038/Put/seqid=0 2024-12-03T06:15:09,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206569070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206569070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206569072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742295_1471 (size=12001) 2024-12-03T06:15:09,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,079 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/26f1c24b8cee4a9cb3d64783d064dae0 2024-12-03T06:15:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206569079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206569080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1c594b7b1f794b5f800c685dfcec98b8 is 50, key is test_row_0/C:col10/1733206508038/Put/seqid=0 2024-12-03T06:15:09,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742296_1472 (size=12001) 2024-12-03T06:15:09,100 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1c594b7b1f794b5f800c685dfcec98b8 2024-12-03T06:15:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/eae35a5e76b64c1d9ca14e9c5fea0a9e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e 2024-12-03T06:15:09,111 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T06:15:09,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/26f1c24b8cee4a9cb3d64783d064dae0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0 2024-12-03T06:15:09,114 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T06:15:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1c594b7b1f794b5f800c685dfcec98b8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8 2024-12-03T06:15:09,118 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T06:15:09,119 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8965fa85a45aa2b83708e0c9d6c60459 in 530ms, sequenceid=78, compaction requested=false 2024-12-03T06:15:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T06:15:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T06:15:09,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T06:15:09,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2970 sec 2024-12-03T06:15:09,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3020 sec 2024-12-03T06:15:09,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:09,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3bfe08f5df734fa4976469ab2c461bf9 is 50, key is test_row_0/A:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742297_1473 (size=12001) 2024-12-03T06:15:09,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3bfe08f5df734fa4976469ab2c461bf9 2024-12-03T06:15:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206569415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206569416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/e888cf01400b48f2947a0bf6b1b33eea is 50, key is test_row_0/B:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206569429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206569430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206569430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742298_1474 (size=12001) 2024-12-03T06:15:09,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/e888cf01400b48f2947a0bf6b1b33eea 2024-12-03T06:15:09,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/edf85e5ad2664b799e69d2ef2285a62c is 50, key is test_row_0/C:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742299_1475 (size=12001) 2024-12-03T06:15:09,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206569543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206569543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206569550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206569550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206569550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206569760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206569760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206569770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206569770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206569770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:09,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/edf85e5ad2664b799e69d2ef2285a62c 2024-12-03T06:15:09,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3bfe08f5df734fa4976469ab2c461bf9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9 2024-12-03T06:15:09,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T06:15:09,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/e888cf01400b48f2947a0bf6b1b33eea as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea 2024-12-03T06:15:09,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T06:15:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/edf85e5ad2664b799e69d2ef2285a62c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c 2024-12-03T06:15:09,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T06:15:09,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8965fa85a45aa2b83708e0c9d6c60459 in 503ms, sequenceid=95, compaction requested=true 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:09,889 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:09,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:09,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:09,890 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:09,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:09,890 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:09,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:09,890 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:09,890 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:09,890 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/b90da0e5442f45e9b3347cca1033fbf4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.3 K 2024-12-03T06:15:09,890 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/045f51503fbc4ae380d230b044b33742, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.3 K 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 045f51503fbc4ae380d230b044b33742, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506914 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90da0e5442f45e9b3347cca1033fbf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506914 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f1c24b8cee4a9cb3d64783d064dae0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733206508038 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae35a5e76b64c1d9ca14e9c5fea0a9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733206508038 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e888cf01400b48f2947a0bf6b1b33eea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:09,891 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfe08f5df734fa4976469ab2c461bf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:09,901 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:09,901 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:09,901 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/22bc9d3faaf04a4480a4fb2b79782bf3 is 50, key is test_row_0/A:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,901 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6ea5b51e06064f89b13cd44b25be9b4c is 50, key is test_row_0/B:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742301_1477 (size=12207) 2024-12-03T06:15:09,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742300_1476 (size=12207) 2024-12-03T06:15:09,925 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6ea5b51e06064f89b13cd44b25be9b4c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ea5b51e06064f89b13cd44b25be9b4c 2024-12-03T06:15:09,930 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 6ea5b51e06064f89b13cd44b25be9b4c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:09,930 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:09,930 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206509889; duration=0sec 2024-12-03T06:15:09,930 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:09,930 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:09,930 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:09,931 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:09,931 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:09,931 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:09,931 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/527e08d957684094801cf276756c7ec0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.3 K 2024-12-03T06:15:09,931 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 527e08d957684094801cf276756c7ec0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733206506914 2024-12-03T06:15:09,932 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c594b7b1f794b5f800c685dfcec98b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733206508038 2024-12-03T06:15:09,932 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting edf85e5ad2664b799e69d2ef2285a62c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:09,938 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:09,938 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/84f27e8010a84959b6a7fbbc84365c15 is 50, key is test_row_0/C:col10/1733206509384/Put/seqid=0 2024-12-03T06:15:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742302_1478 (size=12207) 2024-12-03T06:15:09,965 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/84f27e8010a84959b6a7fbbc84365c15 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/84f27e8010a84959b6a7fbbc84365c15 2024-12-03T06:15:09,969 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into 84f27e8010a84959b6a7fbbc84365c15(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:09,969 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:09,969 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206509889; duration=0sec 2024-12-03T06:15:09,969 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:09,969 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T06:15:09,970 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:09,970 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T06:15:09,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:09,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:09,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:10,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6c5b82afa1ce4384870e297066f07f99 is 50, key is test_row_0/A:col10/1733206510073/Put/seqid=0 2024-12-03T06:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206570092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206570094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206570096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206570098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206570100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742303_1479 (size=14341) 2024-12-03T06:15:10,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6c5b82afa1ce4384870e297066f07f99 2024-12-03T06:15:10,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d669ce7aec34430eac5fe4286dfdc184 is 50, key is test_row_0/B:col10/1733206510073/Put/seqid=0 2024-12-03T06:15:10,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742304_1480 (size=12001) 2024-12-03T06:15:10,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d669ce7aec34430eac5fe4286dfdc184 2024-12-03T06:15:10,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/162d624102cf4b91af6738d976fcf7e7 is 50, key is test_row_0/C:col10/1733206510073/Put/seqid=0 2024-12-03T06:15:10,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742305_1481 (size=12001) 2024-12-03T06:15:10,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/162d624102cf4b91af6738d976fcf7e7 2024-12-03T06:15:10,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6c5b82afa1ce4384870e297066f07f99 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99 2024-12-03T06:15:10,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99, entries=200, sequenceid=116, filesize=14.0 K 2024-12-03T06:15:10,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d669ce7aec34430eac5fe4286dfdc184 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184 2024-12-03T06:15:10,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184, entries=150, sequenceid=116, filesize=11.7 K 2024-12-03T06:15:10,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/162d624102cf4b91af6738d976fcf7e7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7 2024-12-03T06:15:10,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7, entries=150, sequenceid=116, filesize=11.7 K 2024-12-03T06:15:10,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8965fa85a45aa2b83708e0c9d6c60459 in 134ms, sequenceid=116, compaction requested=false 2024-12-03T06:15:10,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:10,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:10,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e0c993fb01f04b14972ff66843e22bab is 50, key is test_row_0/A:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:10,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742306_1482 (size=12051) 2024-12-03T06:15:10,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206570251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206570254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206570256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206570256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206570258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:10,323 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/22bc9d3faaf04a4480a4fb2b79782bf3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/22bc9d3faaf04a4480a4fb2b79782bf3 2024-12-03T06:15:10,327 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 22bc9d3faaf04a4480a4fb2b79782bf3(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:10,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:10,327 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206509889; duration=0sec 2024-12-03T06:15:10,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:10,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:10,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206570370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206570379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206570379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206570379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206570379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206570589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206570589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206570589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206570589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206570590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:10,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e0c993fb01f04b14972ff66843e22bab 2024-12-03T06:15:10,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/8e4cc01a41ce4e538e52eccc18a9fef9 is 50, key is test_row_0/B:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:10,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742307_1483 (size=12051) 2024-12-03T06:15:10,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/8e4cc01a41ce4e538e52eccc18a9fef9 2024-12-03T06:15:10,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1ccc9e9ce67a48bbadf4d4815f054538 is 50, key is test_row_0/C:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:10,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742308_1484 (size=12051) 2024-12-03T06:15:10,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:10,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:10,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206570899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206570900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206570900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206570910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:10,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206570910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:11,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:11,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:11,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:11,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1ccc9e9ce67a48bbadf4d4815f054538 2024-12-03T06:15:11,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e0c993fb01f04b14972ff66843e22bab as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab 2024-12-03T06:15:11,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T06:15:11,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/8e4cc01a41ce4e538e52eccc18a9fef9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9 2024-12-03T06:15:11,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T06:15:11,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1ccc9e9ce67a48bbadf4d4815f054538 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538 2024-12-03T06:15:11,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T06:15:11,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8965fa85a45aa2b83708e0c9d6c60459 in 876ms, sequenceid=130, compaction requested=true 2024-12-03T06:15:11,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:11,092 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:15:11,094 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:11,095 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:11,095 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:11,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:11,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:11,095 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ea5b51e06064f89b13cd44b25be9b4c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.4 K 2024-12-03T06:15:11,095 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/22bc9d3faaf04a4480a4fb2b79782bf3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=37.7 K 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea5b51e06064f89b13cd44b25be9b4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 22bc9d3faaf04a4480a4fb2b79782bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d669ce7aec34430eac5fe4286dfdc184, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206509427 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5b82afa1ce4384870e297066f07f99, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206509414 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e4cc01a41ce4e538e52eccc18a9fef9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:11,096 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c993fb01f04b14972ff66843e22bab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:11,112 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:11,112 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/95f73e497afe405392a69dcc88d57821 is 50, key is test_row_0/B:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:11,115 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:11,116 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/17daf320b1b146228b7c8b7044c6613b is 50, key is test_row_0/A:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:11,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742310_1486 (size=12359) 2024-12-03T06:15:11,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742309_1485 (size=12359) 2024-12-03T06:15:11,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T06:15:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,206 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:15:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9d90b9adab3e4f59a04eb6a824fd3afd is 50, key is test_row_0/A:col10/1733206510239/Put/seqid=0 2024-12-03T06:15:11,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742311_1487 (size=12151) 2024-12-03T06:15:11,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:11,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206571446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206571447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206571447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206571451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206571451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,552 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/17daf320b1b146228b7c8b7044c6613b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/17daf320b1b146228b7c8b7044c6613b 2024-12-03T06:15:11,556 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 17daf320b1b146228b7c8b7044c6613b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:11,556 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:11,556 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206511092; duration=0sec 2024-12-03T06:15:11,556 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:11,556 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:11,556 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:11,557 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:11,557 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:11,557 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:11,557 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/84f27e8010a84959b6a7fbbc84365c15, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.4 K 2024-12-03T06:15:11,558 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f27e8010a84959b6a7fbbc84365c15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733206508748 2024-12-03T06:15:11,558 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 162d624102cf4b91af6738d976fcf7e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733206509427 2024-12-03T06:15:11,558 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccc9e9ce67a48bbadf4d4815f054538, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:11,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206571559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,575 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/95f73e497afe405392a69dcc88d57821 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/95f73e497afe405392a69dcc88d57821 2024-12-03T06:15:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206571569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206571570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206571570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206571570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,585 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 95f73e497afe405392a69dcc88d57821(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:11,585 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:11,585 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206511093; duration=0sec 2024-12-03T06:15:11,585 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:11,585 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:11,585 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:11,586 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/c6181ab600c54b69a3444d5c32700b6e is 50, key is test_row_0/C:col10/1733206510095/Put/seqid=0 2024-12-03T06:15:11,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742312_1488 (size=12359) 2024-12-03T06:15:11,660 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9d90b9adab3e4f59a04eb6a824fd3afd 2024-12-03T06:15:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/fa3f4adaee6d46ed8f49fc117bd0b13b is 50, key is test_row_0/B:col10/1733206510239/Put/seqid=0 2024-12-03T06:15:11,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742313_1489 (size=12151) 2024-12-03T06:15:11,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206571769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206571789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206571790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206571790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206571790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:11,994 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/c6181ab600c54b69a3444d5c32700b6e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c6181ab600c54b69a3444d5c32700b6e 2024-12-03T06:15:11,999 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into c6181ab600c54b69a3444d5c32700b6e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:11,999 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:11,999 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206511093; duration=0sec 2024-12-03T06:15:11,999 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:11,999 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:12,072 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/fa3f4adaee6d46ed8f49fc117bd0b13b 2024-12-03T06:15:12,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/bb4a71c5a30d4bfd93e2ef078aaf6387 is 50, key is test_row_0/C:col10/1733206510239/Put/seqid=0 2024-12-03T06:15:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742314_1490 (size=12151) 2024-12-03T06:15:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206572079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206572100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206572100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206572100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206572100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:12,484 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/bb4a71c5a30d4bfd93e2ef078aaf6387 2024-12-03T06:15:12,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9d90b9adab3e4f59a04eb6a824fd3afd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd 2024-12-03T06:15:12,492 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T06:15:12,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/fa3f4adaee6d46ed8f49fc117bd0b13b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b 2024-12-03T06:15:12,496 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T06:15:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/bb4a71c5a30d4bfd93e2ef078aaf6387 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387 2024-12-03T06:15:12,506 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T06:15:12,507 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8965fa85a45aa2b83708e0c9d6c60459 in 1301ms, sequenceid=149, compaction requested=false 2024-12-03T06:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T06:15:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T06:15:12,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T06:15:12,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5340 sec 2024-12-03T06:15:12,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.5410 sec 2024-12-03T06:15:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:12,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:12,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:12,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:12,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:12,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:12,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 is 50, key is test_row_0/A:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:12,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742315_1491 (size=14541) 2024-12-03T06:15:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206572637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206572638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206572648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206572646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206572648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206572760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206572760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206572760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206572760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206572760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206572970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206572970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206572970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206572970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206572970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 2024-12-03T06:15:13,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/18dc01ad85a248f7a74446a1943d50e3 is 50, key is test_row_0/B:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:13,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742316_1492 (size=12151) 2024-12-03T06:15:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206573290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206573290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206573290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206573290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206573290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/18dc01ad85a248f7a74446a1943d50e3 2024-12-03T06:15:13,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/14bf4047e6f44626ab708676deca8dca is 50, key is test_row_0/C:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:13,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742317_1493 (size=12151) 2024-12-03T06:15:13,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206573799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206573799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206573800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206573810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206573810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:13,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/14bf4047e6f44626ab708676deca8dca 2024-12-03T06:15:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 2024-12-03T06:15:13,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1, entries=200, sequenceid=168, filesize=14.2 K 2024-12-03T06:15:13,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/18dc01ad85a248f7a74446a1943d50e3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3 2024-12-03T06:15:13,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3, entries=150, sequenceid=168, filesize=11.9 K 2024-12-03T06:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/14bf4047e6f44626ab708676deca8dca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca 2024-12-03T06:15:13,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca, entries=150, sequenceid=168, filesize=11.9 K 2024-12-03T06:15:13,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8965fa85a45aa2b83708e0c9d6c60459 in 1256ms, sequenceid=168, compaction requested=true 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:13,849 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:15:13,849 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:13,850 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:13,850 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:13,850 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:13,850 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/95f73e497afe405392a69dcc88d57821, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.8 K 2024-12-03T06:15:13,851 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:13,851 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:13,851 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:13,851 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/17daf320b1b146228b7c8b7044c6613b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=38.1 K 2024-12-03T06:15:13,853 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f73e497afe405392a69dcc88d57821, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:13,856 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17daf320b1b146228b7c8b7044c6613b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:13,856 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3f4adaee6d46ed8f49fc117bd0b13b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733206510239 2024-12-03T06:15:13,856 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d90b9adab3e4f59a04eb6a824fd3afd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733206510239 2024-12-03T06:15:13,856 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dc01ad85a248f7a74446a1943d50e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511447 2024-12-03T06:15:13,857 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7624b384c14ee4a5d6d5ca43ba8ff1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511445 2024-12-03T06:15:13,870 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:13,870 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/63c226b6f4ee4da49777a8096b9082e1 is 50, key is test_row_0/A:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:13,873 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:13,873 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/71cff6a4d4db4c05ac8fe5edaead57ef is 50, key is test_row_0/B:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742319_1495 (size=12561) 2024-12-03T06:15:13,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742318_1494 (size=12561) 2024-12-03T06:15:13,932 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/71cff6a4d4db4c05ac8fe5edaead57ef as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/71cff6a4d4db4c05ac8fe5edaead57ef 2024-12-03T06:15:13,935 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 71cff6a4d4db4c05ac8fe5edaead57ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:13,935 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:13,935 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206513849; duration=0sec 2024-12-03T06:15:13,935 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:13,935 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:13,935 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:13,938 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:13,938 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:13,938 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:13,938 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c6181ab600c54b69a3444d5c32700b6e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=35.8 K 2024-12-03T06:15:13,938 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c6181ab600c54b69a3444d5c32700b6e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733206510095 2024-12-03T06:15:13,939 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4a71c5a30d4bfd93e2ef078aaf6387, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733206510239 2024-12-03T06:15:13,939 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 14bf4047e6f44626ab708676deca8dca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511447 2024-12-03T06:15:13,950 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:13,951 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b2605b71e0024c15b4c7ca16be4e6f6f is 50, key is test_row_0/C:col10/1733206511447/Put/seqid=0 2024-12-03T06:15:13,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742320_1496 (size=12561) 2024-12-03T06:15:13,980 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b2605b71e0024c15b4c7ca16be4e6f6f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2605b71e0024c15b4c7ca16be4e6f6f 2024-12-03T06:15:13,984 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into b2605b71e0024c15b4c7ca16be4e6f6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:13,984 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:13,984 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206513849; duration=0sec 2024-12-03T06:15:13,984 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:13,984 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T06:15:14,139 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:14,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-03T06:15:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T06:15:14,141 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:14,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:14,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T06:15:14,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:14,295 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/89798483a33049ef83a5cba6e991a89f is 50, key is test_row_0/A:col10/1733206512633/Put/seqid=0 2024-12-03T06:15:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742321_1497 (size=12151) 2024-12-03T06:15:14,324 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/63c226b6f4ee4da49777a8096b9082e1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/63c226b6f4ee4da49777a8096b9082e1 2024-12-03T06:15:14,327 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 63c226b6f4ee4da49777a8096b9082e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:14,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:14,327 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206513849; duration=0sec 2024-12-03T06:15:14,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:14,327 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T06:15:14,704 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/89798483a33049ef83a5cba6e991a89f 2024-12-03T06:15:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/1c503be547f7404783a7948e6c0211d6 is 50, key is test_row_0/B:col10/1733206512633/Put/seqid=0 2024-12-03T06:15:14,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742322_1498 (size=12151) 2024-12-03T06:15:14,732 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/1c503be547f7404783a7948e6c0211d6 2024-12-03T06:15:14,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/31e03a2d4bf14b699b4116064444fff1 is 50, key is test_row_0/C:col10/1733206512633/Put/seqid=0 2024-12-03T06:15:14,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742323_1499 (size=12151) 2024-12-03T06:15:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T06:15:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:14,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206574829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,839 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206574832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206574835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206574836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206574837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206574949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206574950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206574950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:14,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206574950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,165 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/31e03a2d4bf14b699b4116064444fff1 2024-12-03T06:15:15,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206575160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206575160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/89798483a33049ef83a5cba6e991a89f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f 2024-12-03T06:15:15,173 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T06:15:15,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/1c503be547f7404783a7948e6c0211d6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6 2024-12-03T06:15:15,176 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T06:15:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206575169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206575169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/31e03a2d4bf14b699b4116064444fff1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1 2024-12-03T06:15:15,185 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T06:15:15,186 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8965fa85a45aa2b83708e0c9d6c60459 in 890ms, sequenceid=187, compaction requested=false 2024-12-03T06:15:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-03T06:15:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-03T06:15:15,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T06:15:15,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-12-03T06:15:15,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0510 sec 2024-12-03T06:15:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T06:15:15,280 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:15,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-03T06:15:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T06:15:15,283 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:15,283 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:15,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T06:15:15,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:15,437 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/744e747d392840599fec12664f6c6fa8 is 50, key is test_row_0/A:col10/1733206514834/Put/seqid=0 2024-12-03T06:15:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742324_1500 (size=12151) 2024-12-03T06:15:15,446 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/744e747d392840599fec12664f6c6fa8 2024-12-03T06:15:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b45b8dcd2688455c911ae8fb444f0072 is 50, key is test_row_0/B:col10/1733206514834/Put/seqid=0 2024-12-03T06:15:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742325_1501 (size=12151) 2024-12-03T06:15:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:15,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206575496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206575497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206575501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206575509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T06:15:15,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206575620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206575620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206575620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206575620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206575840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206575840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206575840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206575840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:15,855 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b45b8dcd2688455c911ae8fb444f0072 2024-12-03T06:15:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/0ba204484fb5464193703e7df57962bf is 50, key is test_row_0/C:col10/1733206514834/Put/seqid=0 2024-12-03T06:15:15,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742326_1502 (size=12151) 2024-12-03T06:15:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T06:15:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206576159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206576159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206576159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206576159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,270 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/0ba204484fb5464193703e7df57962bf 2024-12-03T06:15:16,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/744e747d392840599fec12664f6c6fa8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8 2024-12-03T06:15:16,279 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8, entries=150, sequenceid=203, filesize=11.9 K 2024-12-03T06:15:16,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/b45b8dcd2688455c911ae8fb444f0072 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072 2024-12-03T06:15:16,284 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072, entries=150, sequenceid=203, filesize=11.9 K 2024-12-03T06:15:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/0ba204484fb5464193703e7df57962bf as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf 2024-12-03T06:15:16,287 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf, entries=150, sequenceid=203, filesize=11.9 K 2024-12-03T06:15:16,288 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8965fa85a45aa2b83708e0c9d6c60459 in 851ms, sequenceid=203, compaction requested=true 2024-12-03T06:15:16,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:16,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:16,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-03T06:15:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-03T06:15:16,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T06:15:16,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-12-03T06:15:16,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0110 sec 2024-12-03T06:15:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T06:15:16,420 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:16,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-12-03T06:15:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T06:15:16,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:16,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:16,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T06:15:16,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:16,576 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3027bdea2f45427bb3ea8e7939ccdda1 is 50, key is test_row_0/A:col10/1733206515493/Put/seqid=0 2024-12-03T06:15:16,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742327_1503 (size=12151) 2024-12-03T06:15:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:16,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206576693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206576694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206576702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206576702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T06:15:16,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206576809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206576809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206576819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206576819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206576850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:16,855 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:16,984 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3027bdea2f45427bb3ea8e7939ccdda1 2024-12-03T06:15:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c66eedd569054454aadc8e4d28e74f19 is 50, key is test_row_0/B:col10/1733206515493/Put/seqid=0 2024-12-03T06:15:16,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742328_1504 (size=12151) 2024-12-03T06:15:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206577019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206577019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206577029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206577030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T06:15:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206577330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206577340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206577350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206577350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,398 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c66eedd569054454aadc8e4d28e74f19 2024-12-03T06:15:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5c03ce2f35a44b9daa8d23f07edaaaca is 50, key is test_row_0/C:col10/1733206515493/Put/seqid=0 2024-12-03T06:15:17,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742329_1505 (size=12151) 2024-12-03T06:15:17,421 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5c03ce2f35a44b9daa8d23f07edaaaca 2024-12-03T06:15:17,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3027bdea2f45427bb3ea8e7939ccdda1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1 2024-12-03T06:15:17,428 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1, entries=150, sequenceid=220, filesize=11.9 K 2024-12-03T06:15:17,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c66eedd569054454aadc8e4d28e74f19 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19 2024-12-03T06:15:17,434 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19, entries=150, sequenceid=220, filesize=11.9 K 2024-12-03T06:15:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5c03ce2f35a44b9daa8d23f07edaaaca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca 2024-12-03T06:15:17,437 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca, entries=150, sequenceid=220, filesize=11.9 K 2024-12-03T06:15:17,438 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8965fa85a45aa2b83708e0c9d6c60459 in 862ms, sequenceid=220, compaction requested=true 2024-12-03T06:15:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-03T06:15:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-03T06:15:17,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T06:15:17,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-12-03T06:15:17,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.0220 sec 2024-12-03T06:15:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T06:15:17,560 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:17,560 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-12-03T06:15:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T06:15:17,564 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:17,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:17,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T06:15:17,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-03T06:15:17,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:17,718 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e59e48c8989247cdacd0bf3f233dc10b is 50, key is test_row_0/A:col10/1733206516690/Put/seqid=0 2024-12-03T06:15:17,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742330_1506 (size=12151) 2024-12-03T06:15:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T06:15:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206577892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206577893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206577894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206577894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206578000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206578009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206578009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206578009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,136 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e59e48c8989247cdacd0bf3f233dc10b 2024-12-03T06:15:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c72f9dee889f434cb95fa2e28efdd86a is 50, key is test_row_0/B:col10/1733206516690/Put/seqid=0 2024-12-03T06:15:18,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742331_1507 (size=12151) 2024-12-03T06:15:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c72f9dee889f434cb95fa2e28efdd86a 2024-12-03T06:15:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T06:15:18,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d6beeeab13e54fee9aebd5d00aabea06 is 50, key is test_row_0/C:col10/1733206516690/Put/seqid=0 2024-12-03T06:15:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742332_1508 (size=12151) 2024-12-03T06:15:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d6beeeab13e54fee9aebd5d00aabea06 2024-12-03T06:15:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e59e48c8989247cdacd0bf3f233dc10b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b 2024-12-03T06:15:18,206 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b, entries=150, sequenceid=235, filesize=11.9 K 2024-12-03T06:15:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c72f9dee889f434cb95fa2e28efdd86a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a 2024-12-03T06:15:18,215 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a, entries=150, sequenceid=235, filesize=11.9 K 2024-12-03T06:15:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206578209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d6beeeab13e54fee9aebd5d00aabea06 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06 2024-12-03T06:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06, entries=150, sequenceid=235, filesize=11.9 K 2024-12-03T06:15:18,226 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8965fa85a45aa2b83708e0c9d6c60459 in 507ms, sequenceid=235, compaction requested=true 2024-12-03T06:15:18,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:18,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:18,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-03T06:15:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-03T06:15:18,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-03T06:15:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:18,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 662 msec 2024-12-03T06:15:18,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:18,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 669 msec 2024-12-03T06:15:18,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3eeb1e29dd8546c08280dd4c5c0ed81c is 50, key is test_row_0/A:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:18,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742333_1509 (size=14541) 2024-12-03T06:15:18,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3eeb1e29dd8546c08280dd4c5c0ed81c 2024-12-03T06:15:18,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/464f716299854de4ad0083d6f62cc463 is 50, key is test_row_0/B:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:18,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742334_1510 (size=12151) 2024-12-03T06:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206578260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206578264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206578274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206578379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206578379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206578379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206578530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206578600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206578600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206578600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/464f716299854de4ad0083d6f62cc463 2024-12-03T06:15:18,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5e7e6c4b897b4ef28e7cefd8831a2c1f is 50, key is test_row_0/C:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:18,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742335_1511 (size=12151) 2024-12-03T06:15:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T06:15:18,700 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:18,700 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-12-03T06:15:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:18,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:18,703 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:18,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:18,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-03T06:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:18,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206578920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206578920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206578920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-03T06:15:19,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:19,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:19,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206579040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5e7e6c4b897b4ef28e7cefd8831a2c1f 2024-12-03T06:15:19,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3eeb1e29dd8546c08280dd4c5c0ed81c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c 2024-12-03T06:15:19,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c, entries=200, sequenceid=254, filesize=14.2 K 2024-12-03T06:15:19,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/464f716299854de4ad0083d6f62cc463 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463 2024-12-03T06:15:19,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T06:15:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/5e7e6c4b897b4ef28e7cefd8831a2c1f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f 2024-12-03T06:15:19,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T06:15:19,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8965fa85a45aa2b83708e0c9d6c60459 in 860ms, sequenceid=254, compaction requested=true 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T06:15:19,089 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T06:15:19,089 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T06:15:19,094 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73316 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T06:15:19,095 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:19,095 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,095 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2605b71e0024c15b4c7ca16be4e6f6f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=71.6 K 2024-12-03T06:15:19,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 75706 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T06:15:19,095 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b2605b71e0024c15b4c7ca16be4e6f6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511447 2024-12-03T06:15:19,095 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:19,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,095 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/63c226b6f4ee4da49777a8096b9082e1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=73.9 K 2024-12-03T06:15:19,108 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c226b6f4ee4da49777a8096b9082e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511447 2024-12-03T06:15:19,108 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e03a2d4bf14b699b4116064444fff1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733206512633 2024-12-03T06:15:19,108 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba204484fb5464193703e7df57962bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733206514827 2024-12-03T06:15:19,109 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89798483a33049ef83a5cba6e991a89f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733206512633 2024-12-03T06:15:19,109 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c03ce2f35a44b9daa8d23f07edaaaca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733206515493 2024-12-03T06:15:19,109 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744e747d392840599fec12664f6c6fa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733206514827 2024-12-03T06:15:19,109 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d6beeeab13e54fee9aebd5d00aabea06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733206516689 2024-12-03T06:15:19,109 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3027bdea2f45427bb3ea8e7939ccdda1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733206515493 2024-12-03T06:15:19,110 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7e6c4b897b4ef28e7cefd8831a2c1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:19,110 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59e48c8989247cdacd0bf3f233dc10b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733206516689 2024-12-03T06:15:19,111 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eeb1e29dd8546c08280dd4c5c0ed81c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:19,129 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:19,130 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/a2efd18ecc1d4e4aa828b113c3f6f257 is 50, key is test_row_0/A:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:19,148 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#436 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:19,149 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/03e73de0517a4bc39078eafe8f27a811 is 50, key is test_row_0/C:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:19,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-03T06:15:19,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,162 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/1e802de1869e44189383c1644137cdea is 50, key is test_row_0/A:col10/1733206518259/Put/seqid=0 2024-12-03T06:15:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742336_1512 (size=12765) 2024-12-03T06:15:19,187 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/a2efd18ecc1d4e4aa828b113c3f6f257 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a2efd18ecc1d4e4aa828b113c3f6f257 2024-12-03T06:15:19,190 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into a2efd18ecc1d4e4aa828b113c3f6f257(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:19,191 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:19,191 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=10, startTime=1733206519089; duration=0sec 2024-12-03T06:15:19,191 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:19,191 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:19,191 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T06:15:19,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73316 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T06:15:19,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:19,201 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:19,201 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/71cff6a4d4db4c05ac8fe5edaead57ef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=71.6 K 2024-12-03T06:15:19,201 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71cff6a4d4db4c05ac8fe5edaead57ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733206511447 2024-12-03T06:15:19,202 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c503be547f7404783a7948e6c0211d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733206512633 2024-12-03T06:15:19,202 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45b8dcd2688455c911ae8fb444f0072, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733206514827 2024-12-03T06:15:19,203 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66eedd569054454aadc8e4d28e74f19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733206515493 2024-12-03T06:15:19,203 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72f9dee889f434cb95fa2e28efdd86a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733206516689 2024-12-03T06:15:19,204 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464f716299854de4ad0083d6f62cc463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742338_1514 (size=12301) 2024-12-03T06:15:19,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742337_1513 (size=12765) 2024-12-03T06:15:19,217 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/03e73de0517a4bc39078eafe8f27a811 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/03e73de0517a4bc39078eafe8f27a811 2024-12-03T06:15:19,223 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into 03e73de0517a4bc39078eafe8f27a811(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:19,223 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:19,223 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=10, startTime=1733206519089; duration=0sec 2024-12-03T06:15:19,223 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:19,223 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:19,241 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#438 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:19,242 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/0e86cc0e3e9945bea27e47dc88118830 is 50, key is test_row_0/B:col10/1733206518227/Put/seqid=0 2024-12-03T06:15:19,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742339_1515 (size=12765) 2024-12-03T06:15:19,280 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/0e86cc0e3e9945bea27e47dc88118830 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e86cc0e3e9945bea27e47dc88118830 2024-12-03T06:15:19,288 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 0e86cc0e3e9945bea27e47dc88118830(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:19,288 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:19,288 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=10, startTime=1733206519089; duration=0sec 2024-12-03T06:15:19,288 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:19,288 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:19,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206579531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206579533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206579534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,607 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/1e802de1869e44189383c1644137cdea 2024-12-03T06:15:19,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/a15ac4a432134d2c8137f7310d81f8fa is 50, key is test_row_0/B:col10/1733206518259/Put/seqid=0 2024-12-03T06:15:19,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742340_1516 (size=12301) 2024-12-03T06:15:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206579649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206579649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206579649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206579859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206579860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206579860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,034 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/a15ac4a432134d2c8137f7310d81f8fa 2024-12-03T06:15:20,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206580060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,062 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d4e564312f7446abb96322e916f7d4ed is 50, key is test_row_0/C:col10/1733206518259/Put/seqid=0 2024-12-03T06:15:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742341_1517 (size=12301) 2024-12-03T06:15:20,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206580169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206580170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206580170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,513 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d4e564312f7446abb96322e916f7d4ed 2024-12-03T06:15:20,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/1e802de1869e44189383c1644137cdea as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea 2024-12-03T06:15:20,519 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T06:15:20,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/a15ac4a432134d2c8137f7310d81f8fa as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa 2024-12-03T06:15:20,523 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T06:15:20,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/d4e564312f7446abb96322e916f7d4ed as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed 2024-12-03T06:15:20,526 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T06:15:20,527 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8965fa85a45aa2b83708e0c9d6c60459 in 1364ms, sequenceid=267, compaction requested=false 2024-12-03T06:15:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-03T06:15:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-03T06:15:20,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T06:15:20,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-12-03T06:15:20,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.8300 sec 2024-12-03T06:15:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:20,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T06:15:20,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:20,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:20,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/223c385f95164c6ea0428c64a9951ff2 is 50, key is test_row_0/A:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:20,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742342_1518 (size=14741) 2024-12-03T06:15:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206580703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206580704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206580705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206580819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206580819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206580819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T06:15:20,850 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:20,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-12-03T06:15:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:20,852 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:20,852 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:20,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:20,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50180 deadline: 1733206580870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:20,875 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:21,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-03T06:15:21,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206581029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206581029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206581029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/223c385f95164c6ea0428c64a9951ff2 2024-12-03T06:15:21,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/74d13621bf8d49bbaafe54137ed18cc7 is 50, key is test_row_0/B:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:21,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742343_1519 (size=12301) 2024-12-03T06:15:21,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/74d13621bf8d49bbaafe54137ed18cc7 2024-12-03T06:15:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/c3a4e3132faa4c3b97a9f6439118377a is 50, key is test_row_0/C:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:21,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742344_1520 (size=12301) 2024-12-03T06:15:21,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-03T06:15:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:21,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-03T06:15:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206581340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206581340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206581340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-03T06:15:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:21,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/c3a4e3132faa4c3b97a9f6439118377a 2024-12-03T06:15:21,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/223c385f95164c6ea0428c64a9951ff2 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2 2024-12-03T06:15:21,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2, entries=200, sequenceid=290, filesize=14.4 K 2024-12-03T06:15:21,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/74d13621bf8d49bbaafe54137ed18cc7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7 2024-12-03T06:15:21,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T06:15:21,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/c3a4e3132faa4c3b97a9f6439118377a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a 2024-12-03T06:15:21,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T06:15:21,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8965fa85a45aa2b83708e0c9d6c60459 in 865ms, sequenceid=290, compaction requested=true 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:21,548 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:21,548 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:21,548 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:21,548 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:21,549 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,549 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,549 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e86cc0e3e9945bea27e47dc88118830, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.5 K 2024-12-03T06:15:21,549 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a2efd18ecc1d4e4aa828b113c3f6f257, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=38.9 K 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2efd18ecc1d4e4aa828b113c3f6f257, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e86cc0e3e9945bea27e47dc88118830, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e802de1869e44189383c1644137cdea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206518259 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a15ac4a432134d2c8137f7310d81f8fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206518259 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d13621bf8d49bbaafe54137ed18cc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519529 2024-12-03T06:15:21,549 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223c385f95164c6ea0428c64a9951ff2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519494 2024-12-03T06:15:21,556 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:21,557 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6ba870a38db44a84ab98d4c0a2118858 is 50, key is test_row_0/B:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:21,559 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:21,559 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ec9f823399bc4a9f957136bf2990a3bb is 50, key is test_row_0/A:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742345_1521 (size=13017) 2024-12-03T06:15:21,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742346_1522 (size=13017) 2024-12-03T06:15:21,572 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6ba870a38db44a84ab98d4c0a2118858 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ba870a38db44a84ab98d4c0a2118858 2024-12-03T06:15:21,572 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ec9f823399bc4a9f957136bf2990a3bb as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec9f823399bc4a9f957136bf2990a3bb 2024-12-03T06:15:21,578 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into ec9f823399bc4a9f957136bf2990a3bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:21,578 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 6ba870a38db44a84ab98d4c0a2118858(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:21,579 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206521548; duration=0sec 2024-12-03T06:15:21,579 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206521548; duration=0sec 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:21,579 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:21,579 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,580 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/03e73de0517a4bc39078eafe8f27a811, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.5 K 2024-12-03T06:15:21,581 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e73de0517a4bc39078eafe8f27a811, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733206517880 2024-12-03T06:15:21,581 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e564312f7446abb96322e916f7d4ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733206518259 2024-12-03T06:15:21,582 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a4e3132faa4c3b97a9f6439118377a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519529 2024-12-03T06:15:21,591 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:21,592 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/e36494f060064b3cb7f12e04751c6239 is 50, key is test_row_0/C:col10/1733206519529/Put/seqid=0 2024-12-03T06:15:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742347_1523 (size=13017) 2024-12-03T06:15:21,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-03T06:15:21,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:21,617 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/fe85bd5c987b47da8327a0dbf302c420 is 50, key is test_row_0/A:col10/1733206520700/Put/seqid=0 2024-12-03T06:15:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742348_1524 (size=12301) 2024-12-03T06:15:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:21,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:21,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206581878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206581879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206581880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206581990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206581990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206581990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,019 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/e36494f060064b3cb7f12e04751c6239 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e36494f060064b3cb7f12e04751c6239 2024-12-03T06:15:22,022 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into e36494f060064b3cb7f12e04751c6239(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:22,022 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:22,022 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206521548; duration=0sec 2024-12-03T06:15:22,023 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:22,023 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:22,040 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/fe85bd5c987b47da8327a0dbf302c420 2024-12-03T06:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c7233719878d44d79ee398a4430e04d0 is 50, key is test_row_0/B:col10/1733206520700/Put/seqid=0 2024-12-03T06:15:22,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742349_1525 (size=12301) 2024-12-03T06:15:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206582089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,093 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206582199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206582199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206582199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,448 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c7233719878d44d79ee398a4430e04d0 2024-12-03T06:15:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/e5d15b2901d949ea9f629388a32c1520 is 50, key is test_row_0/C:col10/1733206520700/Put/seqid=0 2024-12-03T06:15:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742350_1526 (size=12301) 2024-12-03T06:15:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206582509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206582509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206582510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:22,873 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/e5d15b2901d949ea9f629388a32c1520 2024-12-03T06:15:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/fe85bd5c987b47da8327a0dbf302c420 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420 2024-12-03T06:15:22,879 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T06:15:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c7233719878d44d79ee398a4430e04d0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0 2024-12-03T06:15:22,882 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T06:15:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/e5d15b2901d949ea9f629388a32c1520 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520 2024-12-03T06:15:22,886 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T06:15:22,887 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8965fa85a45aa2b83708e0c9d6c60459 in 1269ms, sequenceid=304, compaction requested=false 2024-12-03T06:15:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-12-03T06:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-12-03T06:15:22,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T06:15:22,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0350 sec 2024-12-03T06:15:22,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.0400 sec 2024-12-03T06:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T06:15:23,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:23,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-12-03T06:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:23,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:23,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:23,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:23,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:23,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e3dba0a52d1646e7875e4cd97dd03d5c is 50, key is test_row_0/A:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742351_1527 (size=14741) 2024-12-03T06:15:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206583035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206583042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206583043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:23,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206583149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206583149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206583160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206583359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206583359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206583370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e3dba0a52d1646e7875e4cd97dd03d5c 2024-12-03T06:15:23,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/831c3fcb3bae413aaffc5f6f83b0d2f9 is 50, key is test_row_0/B:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742352_1528 (size=12301) 2024-12-03T06:15:23,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/831c3fcb3bae413aaffc5f6f83b0d2f9 2024-12-03T06:15:23,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/60bc4aa0567b43269d604c3d652fef1b is 50, key is test_row_0/C:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742353_1529 (size=12301) 2024-12-03T06:15:23,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206583670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206583670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206583680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:23,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:23,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/60bc4aa0567b43269d604c3d652fef1b 2024-12-03T06:15:23,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/e3dba0a52d1646e7875e4cd97dd03d5c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c 2024-12-03T06:15:23,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c, entries=200, sequenceid=326, filesize=14.4 K 2024-12-03T06:15:23,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/831c3fcb3bae413aaffc5f6f83b0d2f9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9 2024-12-03T06:15:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T06:15:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/60bc4aa0567b43269d604c3d652fef1b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b 2024-12-03T06:15:23,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T06:15:23,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8965fa85a45aa2b83708e0c9d6c60459 in 842ms, sequenceid=326, compaction requested=true 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:23,864 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:23,864 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:23,865 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,865 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,865 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec9f823399bc4a9f957136bf2990a3bb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=39.1 K 2024-12-03T06:15:23,865 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ba870a38db44a84ab98d4c0a2118858, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.7 K 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba870a38db44a84ab98d4c0a2118858, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519529 2024-12-03T06:15:23,865 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9f823399bc4a9f957136bf2990a3bb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519529 2024-12-03T06:15:23,866 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe85bd5c987b47da8327a0dbf302c420, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733206520700 2024-12-03T06:15:23,866 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c7233719878d44d79ee398a4430e04d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733206520700 2024-12-03T06:15:23,866 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 831c3fcb3bae413aaffc5f6f83b0d2f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:23,866 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3dba0a52d1646e7875e4cd97dd03d5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:23,871 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:23,871 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:23,871 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3c9b3866bbc446ee88b3019959090790 is 50, key is test_row_0/A:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,871 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/11e2b1f383fe41c4a2aa8e24fdcc153d is 50, key is test_row_0/B:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742354_1530 (size=13119) 2024-12-03T06:15:23,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742355_1531 (size=13119) 2024-12-03T06:15:23,885 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/11e2b1f383fe41c4a2aa8e24fdcc153d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/11e2b1f383fe41c4a2aa8e24fdcc153d 2024-12-03T06:15:23,889 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into 11e2b1f383fe41c4a2aa8e24fdcc153d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:23,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:23,889 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206523864; duration=0sec 2024-12-03T06:15:23,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:23,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:23,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:23,889 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:23,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:23,890 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,890 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e36494f060064b3cb7f12e04751c6239, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.7 K 2024-12-03T06:15:23,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e36494f060064b3cb7f12e04751c6239, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733206519529 2024-12-03T06:15:23,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d15b2901d949ea9f629388a32c1520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733206520700 2024-12-03T06:15:23,890 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bc4aa0567b43269d604c3d652fef1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:23,895 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:23,895 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/7ac78144362f4aff8c794fe5ca411b8d is 50, key is test_row_0/C:col10/1733206521877/Put/seqid=0 2024-12-03T06:15:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742356_1532 (size=13119) 2024-12-03T06:15:23,901 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/7ac78144362f4aff8c794fe5ca411b8d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/7ac78144362f4aff8c794fe5ca411b8d 2024-12-03T06:15:23,905 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into 7ac78144362f4aff8c794fe5ca411b8d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:23,905 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:23,905 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206523864; duration=0sec 2024-12-03T06:15:23,905 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:23,905 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:23,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-03T06:15:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:23,918 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:15:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9e9533083c1549a6b8f3527210e3d932 is 50, key is test_row_0/A:col10/1733206523032/Put/seqid=0 2024-12-03T06:15:23,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742357_1533 (size=12301) 2024-12-03T06:15:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206584220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206524207, max timestamp=1733206524224Num uniques 859; 2024-12-03T06:15:24,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:15:24,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206524207, max timestamp=1733206524224Num uniques 859; 2024-12-03T06:15:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:15:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733206524207, max timestamp=1733206524224Num uniques 859; 2024-12-03T06:15:24,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T06:15:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206584267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,287 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/3c9b3866bbc446ee88b3019959090790 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3c9b3866bbc446ee88b3019959090790 2024-12-03T06:15:24,290 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 3c9b3866bbc446ee88b3019959090790(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:24,290 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:24,290 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206523864; duration=0sec 2024-12-03T06:15:24,290 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:24,290 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:24,325 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9e9533083c1549a6b8f3527210e3d932 2024-12-03T06:15:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/5cf1d55219464c6dbb0ba10a7728dc92 is 50, key is test_row_0/B:col10/1733206523032/Put/seqid=0 2024-12-03T06:15:24,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206584331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742358_1534 (size=12301) 2024-12-03T06:15:24,373 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/5cf1d55219464c6dbb0ba10a7728dc92 2024-12-03T06:15:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206584380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206584380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b4450caf06164b51a7192e3852294973 is 50, key is test_row_0/C:col10/1733206523032/Put/seqid=0 2024-12-03T06:15:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742359_1535 (size=12301) 2024-12-03T06:15:24,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206584550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206584601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206584601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,839 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b4450caf06164b51a7192e3852294973 2024-12-03T06:15:24,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/9e9533083c1549a6b8f3527210e3d932 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932 2024-12-03T06:15:24,845 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932, entries=150, sequenceid=339, filesize=12.0 K 2024-12-03T06:15:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/5cf1d55219464c6dbb0ba10a7728dc92 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92 2024-12-03T06:15:24,849 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92, entries=150, sequenceid=339, filesize=12.0 K 2024-12-03T06:15:24,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b4450caf06164b51a7192e3852294973 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973 2024-12-03T06:15:24,852 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973, entries=150, sequenceid=339, filesize=12.0 K 2024-12-03T06:15:24,853 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8965fa85a45aa2b83708e0c9d6c60459 in 935ms, sequenceid=339, compaction requested=false 2024-12-03T06:15:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-12-03T06:15:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-12-03T06:15:24,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-03T06:15:24,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-12-03T06:15:24,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 1.8560 sec 2024-12-03T06:15:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:24,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:24,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:24,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ef3ef4fe4f8548f8a8b84d6eeba6110b is 50, key is test_row_0/A:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742360_1536 (size=14741) 2024-12-03T06:15:24,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ef3ef4fe4f8548f8a8b84d6eeba6110b 2024-12-03T06:15:24,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d33ea4ac39e6403980c8c543285bec92 is 50, key is test_row_0/B:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:24,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742361_1537 (size=12301) 2024-12-03T06:15:24,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d33ea4ac39e6403980c8c543285bec92 2024-12-03T06:15:24,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/a3b8ef2d507b42daa413c95d519f9ecd is 50, key is test_row_0/C:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206584917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206584918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206584923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742362_1538 (size=12301) 2024-12-03T06:15:24,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/a3b8ef2d507b42daa413c95d519f9ecd 2024-12-03T06:15:24,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ef3ef4fe4f8548f8a8b84d6eeba6110b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b 2024-12-03T06:15:24,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b, entries=200, sequenceid=376, filesize=14.4 K 2024-12-03T06:15:24,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/d33ea4ac39e6403980c8c543285bec92 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92 2024-12-03T06:15:24,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92, entries=150, sequenceid=376, filesize=12.0 K 2024-12-03T06:15:24,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/a3b8ef2d507b42daa413c95d519f9ecd as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd 2024-12-03T06:15:24,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd, entries=150, sequenceid=376, filesize=12.0 K 2024-12-03T06:15:24,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 8965fa85a45aa2b83708e0c9d6c60459 in 76ms, sequenceid=376, compaction requested=true 2024-12-03T06:15:24,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:24,952 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:24,952 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:24,954 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:24,954 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:24,954 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3c9b3866bbc446ee88b3019959090790, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=39.2 K 2024-12-03T06:15:24,954 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/11e2b1f383fe41c4a2aa8e24fdcc153d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.8 K 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c9b3866bbc446ee88b3019959090790, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:24,954 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e2b1f383fe41c4a2aa8e24fdcc153d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:24,955 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e9533083c1549a6b8f3527210e3d932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733206523032 2024-12-03T06:15:24,955 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf1d55219464c6dbb0ba10a7728dc92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733206523032 2024-12-03T06:15:24,955 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3ef4fe4f8548f8a8b84d6eeba6110b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:24,955 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d33ea4ac39e6403980c8c543285bec92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:24,963 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:24,963 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:24,964 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/f8fa4b8a743b4b86a3d28e91380de3a0 is 50, key is test_row_0/B:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:24,964 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/31be98b650754412b32ddb197bc0aa6d is 50, key is test_row_0/A:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:24,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742364_1540 (size=13221) 2024-12-03T06:15:24,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742363_1539 (size=13221) 2024-12-03T06:15:24,976 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/f8fa4b8a743b4b86a3d28e91380de3a0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/f8fa4b8a743b4b86a3d28e91380de3a0 2024-12-03T06:15:24,983 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into f8fa4b8a743b4b86a3d28e91380de3a0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:24,983 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:24,983 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206524952; duration=0sec 2024-12-03T06:15:24,983 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:24,983 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:24,983 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:24,986 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:24,986 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:24,986 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:24,986 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/7ac78144362f4aff8c794fe5ca411b8d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.8 K 2024-12-03T06:15:24,988 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac78144362f4aff8c794fe5ca411b8d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733206521877 2024-12-03T06:15:24,988 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b4450caf06164b51a7192e3852294973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733206523032 2024-12-03T06:15:24,988 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b8ef2d507b42daa413c95d519f9ecd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:25,000 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:25,000 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b2cd90d3b53740d08339f4e535a4ccb8 is 50, key is test_row_0/C:col10/1733206524243/Put/seqid=0 2024-12-03T06:15:25,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742365_1541 (size=13221) 2024-12-03T06:15:25,022 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/b2cd90d3b53740d08339f4e535a4ccb8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2cd90d3b53740d08339f4e535a4ccb8 2024-12-03T06:15:25,028 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into b2cd90d3b53740d08339f4e535a4ccb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:25,028 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:25,028 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206524952; duration=0sec 2024-12-03T06:15:25,028 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:25,029 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:25,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:25,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6abd98cbc2c2448e9bf76a4974dc23c6 is 50, key is test_row_0/A:col10/1733206525030/Put/seqid=0 2024-12-03T06:15:25,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742366_1542 (size=14741) 2024-12-03T06:15:25,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6abd98cbc2c2448e9bf76a4974dc23c6 2024-12-03T06:15:25,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c339527d87484426b97ef99acbf16177 is 50, key is test_row_0/B:col10/1733206525030/Put/seqid=0 2024-12-03T06:15:25,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742367_1543 (size=12301) 2024-12-03T06:15:25,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c339527d87484426b97ef99acbf16177 2024-12-03T06:15:25,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/693fae153d62470b87203732394e7a7d is 50, key is test_row_0/C:col10/1733206525030/Put/seqid=0 2024-12-03T06:15:25,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742368_1544 (size=12301) 2024-12-03T06:15:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206585087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206525065, max timestamp=1733206525093Num uniques 859; 2024-12-03T06:15:25,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206525065, max timestamp=1733206525093Num uniques 859; 2024-12-03T06:15:25,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:15:25,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:15:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733206525065, max timestamp=1733206525093Num uniques 859; 2024-12-03T06:15:25,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T06:15:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-03T06:15:25,150 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:25,150 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-12-03T06:15:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:25,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:25,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:25,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:25,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206585164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206585166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206585200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206585282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206585290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,382 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/31be98b650754412b32ddb197bc0aa6d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/31be98b650754412b32ddb197bc0aa6d 2024-12-03T06:15:25,388 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 31be98b650754412b32ddb197bc0aa6d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:25,388 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:25,388 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206524951; duration=0sec 2024-12-03T06:15:25,388 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:25,388 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206585420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:25,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/693fae153d62470b87203732394e7a7d 2024-12-03T06:15:25,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/6abd98cbc2c2448e9bf76a4974dc23c6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6 2024-12-03T06:15:25,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6, entries=200, sequenceid=389, filesize=14.4 K 2024-12-03T06:15:25,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/c339527d87484426b97ef99acbf16177 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177 2024-12-03T06:15:25,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177, entries=150, sequenceid=389, filesize=12.0 K 2024-12-03T06:15:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/693fae153d62470b87203732394e7a7d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d 2024-12-03T06:15:25,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206585500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d, entries=150, sequenceid=389, filesize=12.0 K 2024-12-03T06:15:25,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8965fa85a45aa2b83708e0c9d6c60459 in 482ms, sequenceid=389, compaction requested=false 2024-12-03T06:15:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:25,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:25,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/efd2759d97e34e7a9e3759c613cdb558 is 50, key is test_row_0/A:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742369_1545 (size=14741) 2024-12-03T06:15:25,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206585582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,669 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-12-03T06:15:25,669 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:25,669 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,670 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:25,673 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-12-03T06:15:25,673 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:25,673 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,673 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:25,678 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-12-03T06:15:25,678 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:25,678 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,678 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:25,683 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-12-03T06:15:25,683 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:25,683 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,683 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:25,683 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-12-03T06:15:25,683 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:25,683 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,683 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:25,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206585700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206585739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:25,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206585809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206585910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:25,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:25,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:25,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/efd2759d97e34e7a9e3759c613cdb558 2024-12-03T06:15:25,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/432da52049b04ea4b86c9944ba99ab4e is 50, key is test_row_0/B:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:25,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742370_1546 (size=12301) 2024-12-03T06:15:26,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50164 deadline: 1733206586129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:26,130 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in 8965fa85a45aa2b83708e0c9d6c60459 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:26,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206586220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:26,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50136 deadline: 1733206586249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:26,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50098 deadline: 1733206586320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:26,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/432da52049b04ea4b86c9944ba99ab4e 2024-12-03T06:15:26,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/72d1ef691e5b4397994ea76c6f6b1fbe is 50, key is test_row_0/C:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:26,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742371_1547 (size=12301) 2024-12-03T06:15:26,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:26,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:26,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50080 deadline: 1733206586730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 , the old value is region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., hostname=122cddc95ec8,35255,1733206381424, seqNum=2 because the exception is null or not the one we care about 2024-12-03T06:15:26,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/72d1ef691e5b4397994ea76c6f6b1fbe 2024-12-03T06:15:26,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/efd2759d97e34e7a9e3759c613cdb558 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558 2024-12-03T06:15:26,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558, entries=200, sequenceid=425, filesize=14.4 K 2024-12-03T06:15:26,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/432da52049b04ea4b86c9944ba99ab4e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e 2024-12-03T06:15:26,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e, entries=150, sequenceid=425, filesize=12.0 K 2024-12-03T06:15:26,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/72d1ef691e5b4397994ea76c6f6b1fbe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe 2024-12-03T06:15:26,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe, entries=150, sequenceid=425, filesize=12.0 K 2024-12-03T06:15:26,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 8965fa85a45aa2b83708e0c9d6c60459 in 1249ms, sequenceid=425, compaction requested=true 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:26,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8965fa85a45aa2b83708e0c9d6c60459:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:26,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:26,768 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/B is initiating minor compaction (all files) 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/A is initiating minor compaction (all files) 2024-12-03T06:15:26,769 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/A in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,769 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/B in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,769 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/31be98b650754412b32ddb197bc0aa6d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=41.7 K 2024-12-03T06:15:26,769 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/f8fa4b8a743b4b86a3d28e91380de3a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.9 K 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31be98b650754412b32ddb197bc0aa6d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting f8fa4b8a743b4b86a3d28e91380de3a0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6abd98cbc2c2448e9bf76a4974dc23c6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733206524900 2024-12-03T06:15:26,769 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c339527d87484426b97ef99acbf16177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733206524900 2024-12-03T06:15:26,770 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd2759d97e34e7a9e3759c613cdb558, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733206525158 2024-12-03T06:15:26,770 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 432da52049b04ea4b86c9944ba99ab4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733206525158 2024-12-03T06:15:26,774 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#A#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:26,774 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:26,774 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/cc740a1ca87b417e848219db6abe1632 is 50, key is test_row_0/B:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:26,774 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/2c3a08b6bc07473799a24d63f970c479 is 50, key is test_row_0/A:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:26,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742373_1549 (size=13323) 2024-12-03T06:15:26,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742372_1548 (size=13323) 2024-12-03T06:15:26,780 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/2c3a08b6bc07473799a24d63f970c479 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2c3a08b6bc07473799a24d63f970c479 2024-12-03T06:15:26,784 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/A of 8965fa85a45aa2b83708e0c9d6c60459 into 2c3a08b6bc07473799a24d63f970c479(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:26,784 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:26,784 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/A, priority=13, startTime=1733206526768; duration=0sec 2024-12-03T06:15:26,784 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:26,784 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:A 2024-12-03T06:15:26,784 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:26,785 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:26,785 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): 8965fa85a45aa2b83708e0c9d6c60459/C is initiating minor compaction (all files) 2024-12-03T06:15:26,785 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8965fa85a45aa2b83708e0c9d6c60459/C in TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,785 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2cd90d3b53740d08339f4e535a4ccb8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp, totalSize=36.9 K 2024-12-03T06:15:26,786 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cd90d3b53740d08339f4e535a4ccb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733206524243 2024-12-03T06:15:26,786 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693fae153d62470b87203732394e7a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733206524900 2024-12-03T06:15:26,786 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d1ef691e5b4397994ea76c6f6b1fbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733206525158 2024-12-03T06:15:26,799 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8965fa85a45aa2b83708e0c9d6c60459#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:26,800 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1189067190644184b0adfa9dc7aa3070 is 50, key is test_row_0/C:col10/1733206525516/Put/seqid=0 2024-12-03T06:15:26,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742374_1550 (size=13323) 2024-12-03T06:15:26,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-03T06:15:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ec84ec3f5147445d92e7a41ca953ffad is 50, key is test_row_0/A:col10/1733206525580/Put/seqid=0 2024-12-03T06:15:26,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742375_1551 (size=12301) 2024-12-03T06:15:27,181 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/cc740a1ca87b417e848219db6abe1632 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/cc740a1ca87b417e848219db6abe1632 2024-12-03T06:15:27,186 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/B of 8965fa85a45aa2b83708e0c9d6c60459 into cc740a1ca87b417e848219db6abe1632(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:27,186 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:27,186 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/B, priority=13, startTime=1733206526768; duration=0sec 2024-12-03T06:15:27,186 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:27,186 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:B 2024-12-03T06:15:27,207 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/1189067190644184b0adfa9dc7aa3070 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1189067190644184b0adfa9dc7aa3070 2024-12-03T06:15:27,210 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8965fa85a45aa2b83708e0c9d6c60459/C of 8965fa85a45aa2b83708e0c9d6c60459 into 1189067190644184b0adfa9dc7aa3070(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:27,210 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:27,210 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., storeName=8965fa85a45aa2b83708e0c9d6c60459/C, priority=13, startTime=1733206526768; duration=0sec 2024-12-03T06:15:27,211 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:27,211 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8965fa85a45aa2b83708e0c9d6c60459:C 2024-12-03T06:15:27,261 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ec84ec3f5147445d92e7a41ca953ffad 2024-12-03T06:15:27,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/4e06a97ae5d94478b5ad15c6853633f3 is 50, key is test_row_0/B:col10/1733206525580/Put/seqid=0 2024-12-03T06:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:27,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. as already flushing 2024-12-03T06:15:27,271 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-12-03T06:15:27,271 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:27,271 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742376_1552 (size=12301) 2024-12-03T06:15:27,271 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:27,272 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/4e06a97ae5d94478b5ad15c6853633f3 2024-12-03T06:15:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4109a7d88b1c41cf9b0dd02db8a58283 is 50, key is test_row_0/C:col10/1733206525580/Put/seqid=0 2024-12-03T06:15:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742377_1553 (size=12301) 2024-12-03T06:15:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:27,341 INFO [Thread-2048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2048. 2024-12-03T06:15:27,341 DEBUG [Thread-2048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:27,341 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,341 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:27,682 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4109a7d88b1c41cf9b0dd02db8a58283 2024-12-03T06:15:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/ec84ec3f5147445d92e7a41ca953ffad as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec84ec3f5147445d92e7a41ca953ffad 2024-12-03T06:15:27,711 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec84ec3f5147445d92e7a41ca953ffad, entries=150, sequenceid=436, filesize=12.0 K 2024-12-03T06:15:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/4e06a97ae5d94478b5ad15c6853633f3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/4e06a97ae5d94478b5ad15c6853633f3 2024-12-03T06:15:27,716 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/4e06a97ae5d94478b5ad15c6853633f3, entries=150, sequenceid=436, filesize=12.0 K 2024-12-03T06:15:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/4109a7d88b1c41cf9b0dd02db8a58283 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4109a7d88b1c41cf9b0dd02db8a58283 2024-12-03T06:15:27,720 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4109a7d88b1c41cf9b0dd02db8a58283, entries=150, sequenceid=436, filesize=12.0 K 2024-12-03T06:15:27,721 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 8965fa85a45aa2b83708e0c9d6c60459 in 874ms, sequenceid=436, compaction requested=false 2024-12-03T06:15:27,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2603): Flush status journal for 8965fa85a45aa2b83708e0c9d6c60459: 2024-12-03T06:15:27,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:27,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-12-03T06:15:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-12-03T06:15:27,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-03T06:15:27,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5640 sec 2024-12-03T06:15:27,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 2.5740 sec 2024-12-03T06:15:27,742 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-12-03T06:15:27,742 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:27,742 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,742 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:27,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T06:15:29,321 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:29,769 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:15:30,911 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-12-03T06:15:30,912 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:30,912 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:30,912 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:30,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:36,171 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-12-03T06:15:36,171 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:15:36,171 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:36,171 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:36,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1386 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4158 rows 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1391 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4170 rows 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1392 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4176 rows 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1393 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4177 rows 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1382 2024-12-03T06:15:36,172 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4146 rows 2024-12-03T06:15:36,172 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:15:36,172 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:15:36,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:36,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:36,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:15:36,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:15:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:36,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206536176"}]},"ts":"1733206536176"} 2024-12-03T06:15:36,178 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:15:36,178 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:15:36,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:15:36,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, UNASSIGN}] 2024-12-03T06:15:36,181 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, UNASSIGN 2024-12-03T06:15:36,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=8965fa85a45aa2b83708e0c9d6c60459, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:36,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, UNASSIGN because future has completed 2024-12-03T06:15:36,185 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:15:36,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:36,338 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:36,338 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:15:36,338 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 8965fa85a45aa2b83708e0c9d6c60459, disabling compactions & flushes 2024-12-03T06:15:36,338 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:36,338 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:36,338 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. after waiting 0 ms 2024-12-03T06:15:36,338 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:36,339 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(2902): Flushing 8965fa85a45aa2b83708e0c9d6c60459 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=A 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=B 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8965fa85a45aa2b83708e0c9d6c60459, store=C 2024-12-03T06:15:36,339 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:36,345 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/a876ab30ad484fdfaf470681d284767f is 50, key is test_row_1/A:col10/1733206527740/Put/seqid=0 2024-12-03T06:15:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742378_1554 (size=9857) 2024-12-03T06:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:36,752 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/a876ab30ad484fdfaf470681d284767f 2024-12-03T06:15:36,758 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 is 50, key is test_row_1/B:col10/1733206527740/Put/seqid=0 2024-12-03T06:15:36,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742379_1555 (size=9857) 2024-12-03T06:15:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:37,164 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 2024-12-03T06:15:37,176 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/522e5402653247cfba6f2d769bcc7c72 is 50, key is test_row_1/C:col10/1733206527740/Put/seqid=0 2024-12-03T06:15:37,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742380_1556 (size=9857) 2024-12-03T06:15:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:37,580 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/522e5402653247cfba6f2d769bcc7c72 2024-12-03T06:15:37,588 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/A/a876ab30ad484fdfaf470681d284767f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a876ab30ad484fdfaf470681d284767f 2024-12-03T06:15:37,593 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a876ab30ad484fdfaf470681d284767f, entries=100, sequenceid=446, filesize=9.6 K 2024-12-03T06:15:37,593 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 2024-12-03T06:15:37,597 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6f88e099a16a4d9bb4cc878bd3e0c8e5, entries=100, sequenceid=446, filesize=9.6 K 2024-12-03T06:15:37,597 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/.tmp/C/522e5402653247cfba6f2d769bcc7c72 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/522e5402653247cfba6f2d769bcc7c72 2024-12-03T06:15:37,600 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/522e5402653247cfba6f2d769bcc7c72, entries=100, sequenceid=446, filesize=9.6 K 2024-12-03T06:15:37,601 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8965fa85a45aa2b83708e0c9d6c60459 in 1263ms, sequenceid=446, compaction requested=true 2024-12-03T06:15:37,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/b90da0e5442f45e9b3347cca1033fbf4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/22bc9d3faaf04a4480a4fb2b79782bf3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/17daf320b1b146228b7c8b7044c6613b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/63c226b6f4ee4da49777a8096b9082e1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a2efd18ecc1d4e4aa828b113c3f6f257, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec9f823399bc4a9f957136bf2990a3bb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3c9b3866bbc446ee88b3019959090790, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/31be98b650754412b32ddb197bc0aa6d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558] to archive 2024-12-03T06:15:37,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:37,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2dad234ec6be4a1d9d8449403952c839 2024-12-03T06:15:37,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/586baa48a6314eb79d2e70f479c9b482 2024-12-03T06:15:37,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/419e3c100d5f43189f6edbac23129aab 2024-12-03T06:15:37,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/b90da0e5442f45e9b3347cca1033fbf4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/b90da0e5442f45e9b3347cca1033fbf4 2024-12-03T06:15:37,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/eae35a5e76b64c1d9ca14e9c5fea0a9e 2024-12-03T06:15:37,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/22bc9d3faaf04a4480a4fb2b79782bf3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/22bc9d3faaf04a4480a4fb2b79782bf3 2024-12-03T06:15:37,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3bfe08f5df734fa4976469ab2c461bf9 2024-12-03T06:15:37,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6c5b82afa1ce4384870e297066f07f99 2024-12-03T06:15:37,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/17daf320b1b146228b7c8b7044c6613b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/17daf320b1b146228b7c8b7044c6613b 2024-12-03T06:15:37,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e0c993fb01f04b14972ff66843e22bab 2024-12-03T06:15:37,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9d90b9adab3e4f59a04eb6a824fd3afd 2024-12-03T06:15:37,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/5d7624b384c14ee4a5d6d5ca43ba8ff1 2024-12-03T06:15:37,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/63c226b6f4ee4da49777a8096b9082e1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/63c226b6f4ee4da49777a8096b9082e1 2024-12-03T06:15:37,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/89798483a33049ef83a5cba6e991a89f 2024-12-03T06:15:37,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/744e747d392840599fec12664f6c6fa8 2024-12-03T06:15:37,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3027bdea2f45427bb3ea8e7939ccdda1 2024-12-03T06:15:37,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e59e48c8989247cdacd0bf3f233dc10b 2024-12-03T06:15:37,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3eeb1e29dd8546c08280dd4c5c0ed81c 2024-12-03T06:15:37,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a2efd18ecc1d4e4aa828b113c3f6f257 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a2efd18ecc1d4e4aa828b113c3f6f257 2024-12-03T06:15:37,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/1e802de1869e44189383c1644137cdea 2024-12-03T06:15:37,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/223c385f95164c6ea0428c64a9951ff2 2024-12-03T06:15:37,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec9f823399bc4a9f957136bf2990a3bb to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec9f823399bc4a9f957136bf2990a3bb 2024-12-03T06:15:37,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/fe85bd5c987b47da8327a0dbf302c420 2024-12-03T06:15:37,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/e3dba0a52d1646e7875e4cd97dd03d5c 2024-12-03T06:15:37,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3c9b3866bbc446ee88b3019959090790 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/3c9b3866bbc446ee88b3019959090790 2024-12-03T06:15:37,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/9e9533083c1549a6b8f3527210e3d932 2024-12-03T06:15:37,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ef3ef4fe4f8548f8a8b84d6eeba6110b 2024-12-03T06:15:37,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/31be98b650754412b32ddb197bc0aa6d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/31be98b650754412b32ddb197bc0aa6d 2024-12-03T06:15:37,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/6abd98cbc2c2448e9bf76a4974dc23c6 2024-12-03T06:15:37,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/efd2759d97e34e7a9e3759c613cdb558 2024-12-03T06:15:37,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/045f51503fbc4ae380d230b044b33742, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ea5b51e06064f89b13cd44b25be9b4c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/95f73e497afe405392a69dcc88d57821, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/71cff6a4d4db4c05ac8fe5edaead57ef, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e86cc0e3e9945bea27e47dc88118830, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ba870a38db44a84ab98d4c0a2118858, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/11e2b1f383fe41c4a2aa8e24fdcc153d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/f8fa4b8a743b4b86a3d28e91380de3a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e] to archive 2024-12-03T06:15:37,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:37,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/72d6a9bebeb349f1a5c7a7fb5b3dfb32 2024-12-03T06:15:37,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b921a38d202542d3b5344035beda5d0a 2024-12-03T06:15:37,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/045f51503fbc4ae380d230b044b33742 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/045f51503fbc4ae380d230b044b33742 2024-12-03T06:15:37,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e0846d713d84878814c62b556a1be7d 2024-12-03T06:15:37,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/26f1c24b8cee4a9cb3d64783d064dae0 2024-12-03T06:15:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ea5b51e06064f89b13cd44b25be9b4c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ea5b51e06064f89b13cd44b25be9b4c 2024-12-03T06:15:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/e888cf01400b48f2947a0bf6b1b33eea 2024-12-03T06:15:37,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d669ce7aec34430eac5fe4286dfdc184 2024-12-03T06:15:37,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/95f73e497afe405392a69dcc88d57821 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/95f73e497afe405392a69dcc88d57821 2024-12-03T06:15:37,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/8e4cc01a41ce4e538e52eccc18a9fef9 2024-12-03T06:15:37,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/fa3f4adaee6d46ed8f49fc117bd0b13b 2024-12-03T06:15:37,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/71cff6a4d4db4c05ac8fe5edaead57ef to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/71cff6a4d4db4c05ac8fe5edaead57ef 2024-12-03T06:15:37,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/18dc01ad85a248f7a74446a1943d50e3 2024-12-03T06:15:37,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/1c503be547f7404783a7948e6c0211d6 2024-12-03T06:15:37,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/b45b8dcd2688455c911ae8fb444f0072 2024-12-03T06:15:37,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c66eedd569054454aadc8e4d28e74f19 2024-12-03T06:15:37,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c72f9dee889f434cb95fa2e28efdd86a 2024-12-03T06:15:37,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e86cc0e3e9945bea27e47dc88118830 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/0e86cc0e3e9945bea27e47dc88118830 2024-12-03T06:15:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/464f716299854de4ad0083d6f62cc463 2024-12-03T06:15:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/a15ac4a432134d2c8137f7310d81f8fa 2024-12-03T06:15:37,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ba870a38db44a84ab98d4c0a2118858 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6ba870a38db44a84ab98d4c0a2118858 2024-12-03T06:15:37,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/74d13621bf8d49bbaafe54137ed18cc7 2024-12-03T06:15:37,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c7233719878d44d79ee398a4430e04d0 2024-12-03T06:15:37,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/11e2b1f383fe41c4a2aa8e24fdcc153d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/11e2b1f383fe41c4a2aa8e24fdcc153d 2024-12-03T06:15:37,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/831c3fcb3bae413aaffc5f6f83b0d2f9 2024-12-03T06:15:37,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/5cf1d55219464c6dbb0ba10a7728dc92 2024-12-03T06:15:37,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/f8fa4b8a743b4b86a3d28e91380de3a0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/f8fa4b8a743b4b86a3d28e91380de3a0 2024-12-03T06:15:37,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/d33ea4ac39e6403980c8c543285bec92 2024-12-03T06:15:37,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/c339527d87484426b97ef99acbf16177 2024-12-03T06:15:37,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/432da52049b04ea4b86c9944ba99ab4e 2024-12-03T06:15:37,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/527e08d957684094801cf276756c7ec0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/84f27e8010a84959b6a7fbbc84365c15, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c6181ab600c54b69a3444d5c32700b6e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2605b71e0024c15b4c7ca16be4e6f6f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/03e73de0517a4bc39078eafe8f27a811, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e36494f060064b3cb7f12e04751c6239, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/7ac78144362f4aff8c794fe5ca411b8d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2cd90d3b53740d08339f4e535a4ccb8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe] to archive 2024-12-03T06:15:37,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:15:37,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4fe02173b663454fa95c802c648338c6 2024-12-03T06:15:37,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b00a92483230499cbb894ae3216c07f3 2024-12-03T06:15:37,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/527e08d957684094801cf276756c7ec0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/527e08d957684094801cf276756c7ec0 2024-12-03T06:15:37,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/15c17831a5264c34925e831790524e8b 2024-12-03T06:15:37,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1c594b7b1f794b5f800c685dfcec98b8 2024-12-03T06:15:37,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/84f27e8010a84959b6a7fbbc84365c15 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/84f27e8010a84959b6a7fbbc84365c15 2024-12-03T06:15:37,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/edf85e5ad2664b799e69d2ef2285a62c 2024-12-03T06:15:37,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/162d624102cf4b91af6738d976fcf7e7 2024-12-03T06:15:37,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c6181ab600c54b69a3444d5c32700b6e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c6181ab600c54b69a3444d5c32700b6e 2024-12-03T06:15:37,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1ccc9e9ce67a48bbadf4d4815f054538 2024-12-03T06:15:37,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/bb4a71c5a30d4bfd93e2ef078aaf6387 2024-12-03T06:15:37,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2605b71e0024c15b4c7ca16be4e6f6f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2605b71e0024c15b4c7ca16be4e6f6f 2024-12-03T06:15:37,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/14bf4047e6f44626ab708676deca8dca 2024-12-03T06:15:37,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/31e03a2d4bf14b699b4116064444fff1 2024-12-03T06:15:37,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/0ba204484fb5464193703e7df57962bf 2024-12-03T06:15:37,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5c03ce2f35a44b9daa8d23f07edaaaca 2024-12-03T06:15:37,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d6beeeab13e54fee9aebd5d00aabea06 2024-12-03T06:15:37,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/03e73de0517a4bc39078eafe8f27a811 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/03e73de0517a4bc39078eafe8f27a811 2024-12-03T06:15:37,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/5e7e6c4b897b4ef28e7cefd8831a2c1f 2024-12-03T06:15:37,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/d4e564312f7446abb96322e916f7d4ed 2024-12-03T06:15:37,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e36494f060064b3cb7f12e04751c6239 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e36494f060064b3cb7f12e04751c6239 2024-12-03T06:15:37,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/c3a4e3132faa4c3b97a9f6439118377a 2024-12-03T06:15:37,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/e5d15b2901d949ea9f629388a32c1520 2024-12-03T06:15:37,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/7ac78144362f4aff8c794fe5ca411b8d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/7ac78144362f4aff8c794fe5ca411b8d 2024-12-03T06:15:37,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/60bc4aa0567b43269d604c3d652fef1b 2024-12-03T06:15:37,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b4450caf06164b51a7192e3852294973 2024-12-03T06:15:37,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2cd90d3b53740d08339f4e535a4ccb8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/b2cd90d3b53740d08339f4e535a4ccb8 2024-12-03T06:15:37,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/a3b8ef2d507b42daa413c95d519f9ecd 2024-12-03T06:15:37,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/693fae153d62470b87203732394e7a7d 2024-12-03T06:15:37,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/72d1ef691e5b4397994ea76c6f6b1fbe 2024-12-03T06:15:37,675 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/recovered.edits/449.seqid, newMaxSeqId=449, maxSeqId=1 2024-12-03T06:15:37,676 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. 2024-12-03T06:15:37,676 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 8965fa85a45aa2b83708e0c9d6c60459: Waiting for close lock at 1733206536338Running coprocessor pre-close hooks at 1733206536338Disabling compacts and flushes for region at 1733206536338Disabling writes for close at 1733206536338Obtaining lock to block concurrent updates at 1733206536339 (+1 ms)Preparing flush snapshotting stores in 8965fa85a45aa2b83708e0c9d6c60459 at 1733206536339Finished memstore snapshotting TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733206536339Flushing stores of TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459. at 1733206536340 (+1 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/A: creating writer at 1733206536340Flushing 8965fa85a45aa2b83708e0c9d6c60459/A: appending metadata at 1733206536345 (+5 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/A: closing flushed file at 1733206536345Flushing 8965fa85a45aa2b83708e0c9d6c60459/B: creating writer at 1733206536755 (+410 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/B: appending metadata at 1733206536758 (+3 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/B: closing flushed file at 1733206536758Flushing 8965fa85a45aa2b83708e0c9d6c60459/C: creating writer at 1733206537172 (+414 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/C: appending metadata at 1733206537175 (+3 ms)Flushing 8965fa85a45aa2b83708e0c9d6c60459/C: closing flushed file at 1733206537175Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cda71c9: reopening flushed file at 1733206537587 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@506c1825: reopening flushed file at 1733206537593 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@434b3638: reopening flushed file at 1733206537597 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8965fa85a45aa2b83708e0c9d6c60459 in 1263ms, sequenceid=446, compaction requested=true at 1733206537601 (+4 ms)Writing region close event to WAL at 1733206537673 (+72 ms)Running coprocessor post-close hooks at 1733206537676 (+3 ms)Closed at 1733206537676 2024-12-03T06:15:37,677 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:37,678 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=8965fa85a45aa2b83708e0c9d6c60459, regionState=CLOSED 2024-12-03T06:15:37,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:37,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-03T06:15:37,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 8965fa85a45aa2b83708e0c9d6c60459, server=122cddc95ec8,35255,1733206381424 in 1.4950 sec 2024-12-03T06:15:37,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T06:15:37,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8965fa85a45aa2b83708e0c9d6c60459, UNASSIGN in 1.5010 sec 2024-12-03T06:15:37,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-03T06:15:37,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5050 sec 2024-12-03T06:15:37,686 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206537685"}]},"ts":"1733206537685"} 2024-12-03T06:15:37,687 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:15:37,688 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:15:37,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-12-03T06:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T06:15:38,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:38,321 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,323 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=155, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-03T06:15:38,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=155, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,327 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:38,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/recovered.edits] 2024-12-03T06:15:38,334 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2c3a08b6bc07473799a24d63f970c479 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/2c3a08b6bc07473799a24d63f970c479 2024-12-03T06:15:38,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a876ab30ad484fdfaf470681d284767f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/a876ab30ad484fdfaf470681d284767f 2024-12-03T06:15:38,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec84ec3f5147445d92e7a41ca953ffad to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/A/ec84ec3f5147445d92e7a41ca953ffad 2024-12-03T06:15:38,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/4e06a97ae5d94478b5ad15c6853633f3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/4e06a97ae5d94478b5ad15c6853633f3 2024-12-03T06:15:38,341 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/6f88e099a16a4d9bb4cc878bd3e0c8e5 2024-12-03T06:15:38,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/cc740a1ca87b417e848219db6abe1632 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/B/cc740a1ca87b417e848219db6abe1632 2024-12-03T06:15:38,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1189067190644184b0adfa9dc7aa3070 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/1189067190644184b0adfa9dc7aa3070 2024-12-03T06:15:38,347 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4109a7d88b1c41cf9b0dd02db8a58283 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/4109a7d88b1c41cf9b0dd02db8a58283 2024-12-03T06:15:38,348 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/522e5402653247cfba6f2d769bcc7c72 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/C/522e5402653247cfba6f2d769bcc7c72 2024-12-03T06:15:38,352 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/recovered.edits/449.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459/recovered.edits/449.seqid 2024-12-03T06:15:38,353 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/8965fa85a45aa2b83708e0c9d6c60459 2024-12-03T06:15:38,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:15:38,356 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=155, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,358 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:15:38,361 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:15:38,362 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=155, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,362 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:15:38,363 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206538363"}]},"ts":"9223372036854775807"} 2024-12-03T06:15:38,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:15:38,365 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8965fa85a45aa2b83708e0c9d6c60459, NAME => 'TestAcidGuarantees,,1733206503462.8965fa85a45aa2b83708e0c9d6c60459.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:15:38,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:15:38,365 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206538365"}]},"ts":"9223372036854775807"} 2024-12-03T06:15:38,367 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:15:38,367 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=155, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-12-03T06:15:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-03T06:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:15:38,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:38,445 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 247), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 424), ProcessCount=11 (was 11), AvailableMemoryMB=11424 (was 12110) 2024-12-03T06:15:38,456 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=11424 2024-12-03T06:15:38,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:15:38,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:15:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:38,460 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:15:38,460 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:38,460 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 156 2024-12-03T06:15:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:38,461 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:15:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742381_1557 (size=1061) 2024-12-03T06:15:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:38,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c 2024-12-03T06:15:38,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742382_1558 (size=53) 2024-12-03T06:15:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:39,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:15:39,279 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b3fa0bec8819326450abec32a5991191, disabling compactions & flushes 2024-12-03T06:15:39,279 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,279 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,279 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. after waiting 0 ms 2024-12-03T06:15:39,279 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,279 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,279 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3fa0bec8819326450abec32a5991191: Waiting for close lock at 1733206539279Disabling compacts and flushes for region at 1733206539279Disabling writes for close at 1733206539279Writing region close event to WAL at 1733206539279Closed at 1733206539279 2024-12-03T06:15:39,282 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:15:39,283 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733206539282"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206539282"}]},"ts":"1733206539282"} 2024-12-03T06:15:39,287 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T06:15:39,288 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:15:39,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206539288"}]},"ts":"1733206539288"} 2024-12-03T06:15:39,290 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T06:15:39,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, ASSIGN}] 2024-12-03T06:15:39,291 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, ASSIGN 2024-12-03T06:15:39,292 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, ASSIGN; state=OFFLINE, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=false 2024-12-03T06:15:39,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:39,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, ASSIGN because future has completed 2024-12-03T06:15:39,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:39,606 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,607 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(7752): Opening region: {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:15:39,607 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,608 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:15:39,608 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(7794): checking encryption for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,608 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(7797): checking classloading for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,611 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,614 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:39,614 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName A 2024-12-03T06:15:39,614 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:39,615 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:39,615 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,617 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:39,617 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName B 2024-12-03T06:15:39,617 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:39,618 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:39,618 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,619 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:39,619 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName C 2024-12-03T06:15:39,619 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:39,620 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:39,620 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,620 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1038): replaying wal for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,621 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,621 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,622 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1048): stopping wal replay for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,622 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1060): Cleaning up temporary data for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,623 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:15:39,624 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1093): writing seq id for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,627 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:15:39,627 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1114): Opened b3fa0bec8819326450abec32a5991191; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74566267, jitterRate=0.1111239641904831}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:15:39,627 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:39,628 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegion(1006): Region open journal for b3fa0bec8819326450abec32a5991191: Running coprocessor pre-open hook at 1733206539608Writing region info on filesystem at 1733206539608Initializing all the Stores at 1733206539610 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206539610Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206539610Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206539610Cleaning up temporary data from old regions at 1733206539622 (+12 ms)Running coprocessor post-open hooks at 1733206539627 (+5 ms)Region opened successfully at 1733206539628 (+1 ms) 2024-12-03T06:15:39,629 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., pid=158, masterSystemTime=1733206539601 2024-12-03T06:15:39,631 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,631 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=158}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:39,632 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:39,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:39,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-12-03T06:15:39,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 in 188 msec 2024-12-03T06:15:39,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T06:15:39,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, ASSIGN in 347 msec 2024-12-03T06:15:39,640 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:15:39,640 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206539640"}]},"ts":"1733206539640"} 2024-12-03T06:15:39,642 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T06:15:39,643 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=156, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:15:39,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-03T06:15:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T06:15:40,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:40,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:40,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:40,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:40,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cc263b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:40,620 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5bfa90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:40,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:40,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:40,624 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,42621,1733206380709 2024-12-03T06:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d324120 2024-12-03T06:15:40,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:15:40,626 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:15:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T06:15:40,627 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:15:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T06:15:40,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742383_1559 (size=1097) 2024-12-03T06:15:41,041 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T06:15:41,042 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T06:15:41,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:15:41,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE}] 2024-12-03T06:15:41,054 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE 2024-12-03T06:15:41,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE because future has completed 2024-12-03T06:15:41,056 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:15:41,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:41,209 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(122): Close b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,209 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:15:41,209 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1722): Closing b3fa0bec8819326450abec32a5991191, disabling compactions & flushes 2024-12-03T06:15:41,209 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,210 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,210 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. after waiting 0 ms 2024-12-03T06:15:41,210 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,216 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:15:41,217 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,217 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1676): Region close journal for b3fa0bec8819326450abec32a5991191: Waiting for close lock at 1733206541209Running coprocessor pre-close hooks at 1733206541209Disabling compacts and flushes for region at 1733206541209Disabling writes for close at 1733206541210 (+1 ms)Writing region close event to WAL at 1733206541211 (+1 ms)Running coprocessor post-close hooks at 1733206541217 (+6 ms)Closed at 1733206541217 2024-12-03T06:15:41,217 WARN [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionServer(3299): Not adding moved region record: b3fa0bec8819326450abec32a5991191 to self. 2024-12-03T06:15:41,220 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(157): Closed b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,220 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=CLOSED 2024-12-03T06:15:41,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:41,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-03T06:15:41,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 in 167 msec 2024-12-03T06:15:41,226 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE; state=CLOSED, location=122cddc95ec8,35255,1733206381424; forceNewPlan=false, retain=true 2024-12-03T06:15:41,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=OPENING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE because future has completed 2024-12-03T06:15:41,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=161, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:15:41,541 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,541 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7752): Opening region: {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:15:41,541 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,541 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:15:41,541 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7794): checking encryption for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,541 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7797): checking classloading for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,543 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,544 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:41,544 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName A 2024-12-03T06:15:41,545 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:41,546 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:41,546 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,547 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:41,547 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName B 2024-12-03T06:15:41,547 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:41,548 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:41,548 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,549 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T06:15:41,549 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3fa0bec8819326450abec32a5991191 columnFamilyName C 2024-12-03T06:15:41,549 DEBUG [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:41,550 INFO [StoreOpener-b3fa0bec8819326450abec32a5991191-1 {}] regionserver.HStore(327): Store=b3fa0bec8819326450abec32a5991191/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:15:41,550 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,550 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1038): replaying wal for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,551 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,553 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,554 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1048): stopping wal replay for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,554 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1060): Cleaning up temporary data for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,555 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T06:15:41,557 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): writing seq id for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,558 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1114): Opened b3fa0bec8819326450abec32a5991191; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75194120, jitterRate=0.12047970294952393}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T06:15:41,558 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,558 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1006): Region open journal for b3fa0bec8819326450abec32a5991191: Running coprocessor pre-open hook at 1733206541541Writing region info on filesystem at 1733206541541Initializing all the Stores at 1733206541542 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206541542Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206541543 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206541543Cleaning up temporary data from old regions at 1733206541554 (+11 ms)Running coprocessor post-open hooks at 1733206541558 (+4 ms)Region opened successfully at 1733206541558 2024-12-03T06:15:41,559 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., pid=163, masterSystemTime=1733206541537 2024-12-03T06:15:41,562 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,562 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,562 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=OPEN, openSeqNum=5, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=161, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:15:41,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=161 2024-12-03T06:15:41,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=161, state=SUCCESS, hasLock=false; OpenRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 in 181 msec 2024-12-03T06:15:41,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T06:15:41,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, REOPEN/MOVE in 513 msec 2024-12-03T06:15:41,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-03T06:15:41,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-12-03T06:15:41,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-12-03T06:15:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-03T06:15:41,574 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:41,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1056708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,576 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861352a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,578 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51df93a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246d7ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,580 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f701a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,582 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c38a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,583 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564dbbfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4e34c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0bb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,587 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43702d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d443db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e4ce43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d571da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ddc5293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529f191a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,597 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545b9404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757cdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,600 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6900ecc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,42621,-1 for getting cluster id 2024-12-03T06:15:41,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:15:41,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ff2d4dc-7837-43e0-8545-df22b9939adf' 2024-12-03T06:15:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:15:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ff2d4dc-7837-43e0-8545-df22b9939adf" 2024-12-03T06:15:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e4ca8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,42621,-1] 2024-12-03T06:15:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:15:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:15:41,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:15:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ebbb66a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:15:41,603 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,603 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,603 DEBUG [Thread-2500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,604 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:41,604 DEBUG [Thread-2485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,605 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,605 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,605 DEBUG [Thread-2483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,606 DEBUG [Thread-2491 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,606 DEBUG [Thread-2491 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-03T06:15:41,607 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,607 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,35255,1733206381424, seqNum=-1] 2024-12-03T06:15:41,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:15:41,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,608 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,609 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:41,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:15:41,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5] 2024-12-03T06:15:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:41,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:41,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203859e5f0825e94149857c7c7ecc53bd12_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206541616/Put/seqid=0 2024-12-03T06:15:41,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742384_1560 (size=12154) 2024-12-03T06:15:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206601641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206541637, max timestamp=1733206541644Num uniques 859; 2024-12-03T06:15:41,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:15:41,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206541637, max timestamp=1733206541644Num uniques 859; 2024-12-03T06:15:41,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733206541637, max timestamp=1733206541644Num uniques 859; 2024-12-03T06:15:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/122cddc95ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T06:15:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206601663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206601664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206601666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206601668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206601749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206601779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206601779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206601780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206601780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:41,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:41,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206601959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206601990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206601990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206601990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206601990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:42,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203859e5f0825e94149857c7c7ecc53bd12_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203859e5f0825e94149857c7c7ecc53bd12_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:42,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/07e09cab2c9d4c929bf7887480e3f5f1, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:42,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/07e09cab2c9d4c929bf7887480e3f5f1 is 175, key is test_row_0/A:col10/1733206541616/Put/seqid=0 2024-12-03T06:15:42,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742385_1561 (size=30955) 2024-12-03T06:15:42,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206602269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206602299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206602299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206602300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206602300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/07e09cab2c9d4c929bf7887480e3f5f1 2024-12-03T06:15:42,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/cb1ed8309dbb45c1b1774705aad33fe7 is 50, key is test_row_0/B:col10/1733206541616/Put/seqid=0 2024-12-03T06:15:42,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742386_1562 (size=12001) 2024-12-03T06:15:42,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:42,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206602779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206602810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206602810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206602810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206602810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:42,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/cb1ed8309dbb45c1b1774705aad33fe7 2024-12-03T06:15:42,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/3e81515d839045909ddecbf6de5540a0 is 50, key is test_row_0/C:col10/1733206541616/Put/seqid=0 2024-12-03T06:15:42,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742387_1563 (size=12001) 2024-12-03T06:15:42,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:42,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:42,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:42,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:43,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,196 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:15:43,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:43,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:43,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:43,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:43,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/3e81515d839045909ddecbf6de5540a0 2024-12-03T06:15:43,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/07e09cab2c9d4c929bf7887480e3f5f1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1 2024-12-03T06:15:43,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1, entries=150, sequenceid=15, filesize=30.2 K 2024-12-03T06:15:43,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/cb1ed8309dbb45c1b1774705aad33fe7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7 2024-12-03T06:15:43,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T06:15:43,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/3e81515d839045909ddecbf6de5540a0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0 2024-12-03T06:15:43,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T06:15:43,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b3fa0bec8819326450abec32a5991191 in 1713ms, sequenceid=15, compaction requested=false 2024-12-03T06:15:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:43,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T06:15:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1148dc619b84938882064bec54e5260_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206541657/Put/seqid=0 2024-12-03T06:15:43,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742388_1564 (size=12154) 2024-12-03T06:15:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:43,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:43,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206603816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206603819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,823 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206603830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206603830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206603830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,832 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:43,832 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:43,833 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_2' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:43,854 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1148dc619b84938882064bec54e5260_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1148dc619b84938882064bec54e5260_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/caafb878bab541d2954e3ea4e5562b96, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/caafb878bab541d2954e3ea4e5562b96 is 175, key is test_row_0/A:col10/1733206541657/Put/seqid=0 2024-12-03T06:15:43,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742389_1565 (size=30955) 2024-12-03T06:15:43,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206603930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206604139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:44,258 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/caafb878bab541d2954e3ea4e5562b96 2024-12-03T06:15:44,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/4a1210c9a81c417b881c086cccb8612a is 50, key is test_row_0/B:col10/1733206541657/Put/seqid=0 2024-12-03T06:15:44,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742390_1566 (size=12001) 2024-12-03T06:15:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206604450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:44,674 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/4a1210c9a81c417b881c086cccb8612a 2024-12-03T06:15:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7eb5217a598b4cd3ba0f83c93bd9e363 is 50, key is test_row_0/C:col10/1733206541657/Put/seqid=0 2024-12-03T06:15:44,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742391_1567 (size=12001) 2024-12-03T06:15:44,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206604959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,089 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7eb5217a598b4cd3ba0f83c93bd9e363 2024-12-03T06:15:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/caafb878bab541d2954e3ea4e5562b96 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96 2024-12-03T06:15:45,095 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96, entries=150, sequenceid=51, filesize=30.2 K 2024-12-03T06:15:45,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/4a1210c9a81c417b881c086cccb8612a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a 2024-12-03T06:15:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T06:15:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7eb5217a598b4cd3ba0f83c93bd9e363 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363 2024-12-03T06:15:45,104 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T06:15:45,105 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b3fa0bec8819326450abec32a5991191 in 1663ms, sequenceid=51, compaction requested=false 2024-12-03T06:15:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-03T06:15:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-03T06:15:45,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-03T06:15:45,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4980 sec 2024-12-03T06:15:45,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.5060 sec 2024-12-03T06:15:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T06:15:45,760 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:45,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-03T06:15:45,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:45,766 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:45,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:45,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:45,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033efbe5801f4a44e99d34a1e377c95ca0_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742392_1568 (size=12154) 2024-12-03T06:15:45,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:45,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033efbe5801f4a44e99d34a1e377c95ca0_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033efbe5801f4a44e99d34a1e377c95ca0_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:45,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/de29e94b125648d2bb73e998561144c6, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:45,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/de29e94b125648d2bb73e998561144c6 is 175, key is test_row_0/A:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:45,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742393_1569 (size=30955) 2024-12-03T06:15:45,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206605874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206605875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206605877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206605881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:45,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:45,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:45,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206605969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,972 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206605989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206605990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206605990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206605990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:46,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:46,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206606200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206606200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206606200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206606200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/de29e94b125648d2bb73e998561144c6 2024-12-03T06:15:46,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/d3c1f302e5384a86aa5c7ad6d007cdb3 is 50, key is test_row_0/B:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:46,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742394_1570 (size=12001) 2024-12-03T06:15:46,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206606509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206606510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206606510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206606510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:46,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:46,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/d3c1f302e5384a86aa5c7ad6d007cdb3 2024-12-03T06:15:46,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/1f74ebd596454d649c4edccbbc35183f is 50, key is test_row_0/C:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:46,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742395_1571 (size=12001) 2024-12-03T06:15:46,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/1f74ebd596454d649c4edccbbc35183f 2024-12-03T06:15:46,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/de29e94b125648d2bb73e998561144c6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6 2024-12-03T06:15:46,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6, entries=150, sequenceid=65, filesize=30.2 K 2024-12-03T06:15:46,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/d3c1f302e5384a86aa5c7ad6d007cdb3 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3 2024-12-03T06:15:46,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T06:15:46,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/1f74ebd596454d649c4edccbbc35183f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f 2024-12-03T06:15:46,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T06:15:46,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b3fa0bec8819326450abec32a5991191 in 883ms, sequenceid=65, compaction requested=true 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:46,735 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:46,735 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:46,736 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:46,736 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:46,736 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/B is initiating minor compaction (all files) 2024-12-03T06:15:46,736 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/A is initiating minor compaction (all files) 2024-12-03T06:15:46,736 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/B in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,736 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/A in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,736 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=90.7 K 2024-12-03T06:15:46,736 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.2 K 2024-12-03T06:15:46,736 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,736 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6] 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e09cab2c9d4c929bf7887480e3f5f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206541615 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1ed8309dbb45c1b1774705aad33fe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206541615 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting caafb878bab541d2954e3ea4e5562b96, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206541657 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1210c9a81c417b881c086cccb8612a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206541657 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting de29e94b125648d2bb73e998561144c6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:46,737 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c1f302e5384a86aa5c7ad6d007cdb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:46,742 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:46,742 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#B#compaction#489 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:46,743 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c23e7464d0f34e1ca447d049f25ad47f is 50, key is test_row_0/B:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:46,744 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120398bf693a42224addbbe8c36a3fd7cefe_b3fa0bec8819326450abec32a5991191 store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:46,745 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120398bf693a42224addbbe8c36a3fd7cefe_b3fa0bec8819326450abec32a5991191, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:46,745 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120398bf693a42224addbbe8c36a3fd7cefe_b3fa0bec8819326450abec32a5991191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:46,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742396_1572 (size=12104) 2024-12-03T06:15:46,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742397_1573 (size=4469) 2024-12-03T06:15:46,768 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c23e7464d0f34e1ca447d049f25ad47f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c23e7464d0f34e1ca447d049f25ad47f 2024-12-03T06:15:46,769 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#A#compaction#490 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:46,770 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/9ae42da0b8114e45a23ab415b03ebe15 is 175, key is test_row_0/A:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742398_1574 (size=31058) 2024-12-03T06:15:46,776 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/B of b3fa0bec8819326450abec32a5991191 into c23e7464d0f34e1ca447d049f25ad47f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:46,776 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:46,776 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/B, priority=13, startTime=1733206546735; duration=0sec 2024-12-03T06:15:46,776 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:46,776 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:B 2024-12-03T06:15:46,776 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:46,777 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:46,777 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/C is initiating minor compaction (all files) 2024-12-03T06:15:46,777 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/C in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,778 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.2 K 2024-12-03T06:15:46,778 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e81515d839045909ddecbf6de5540a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733206541615 2024-12-03T06:15:46,778 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb5217a598b4cd3ba0f83c93bd9e363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733206541657 2024-12-03T06:15:46,778 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f74ebd596454d649c4edccbbc35183f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:46,784 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:46,784 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/789448c7c153409c99239caee331787a is 50, key is test_row_0/C:col10/1733206545851/Put/seqid=0 2024-12-03T06:15:46,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742399_1575 (size=12104) 2024-12-03T06:15:46,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:46,836 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033bbb975cfdbf4d74ba4bfebe2397ce6b_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206545873/Put/seqid=0 2024-12-03T06:15:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742400_1576 (size=12154) 2024-12-03T06:15:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:46,860 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033bbb975cfdbf4d74ba4bfebe2397ce6b_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033bbb975cfdbf4d74ba4bfebe2397ce6b_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/55b4bd0e16d74a3e9f8201f4addffc0b, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/55b4bd0e16d74a3e9f8201f4addffc0b is 175, key is test_row_0/A:col10/1733206545873/Put/seqid=0 2024-12-03T06:15:46,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742401_1577 (size=30955) 2024-12-03T06:15:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:47,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:47,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206607030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206607030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206607034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206607139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206607139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206607139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,179 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/9ae42da0b8114e45a23ab415b03ebe15 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15 2024-12-03T06:15:47,183 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/A of b3fa0bec8819326450abec32a5991191 into 9ae42da0b8114e45a23ab415b03ebe15(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:47,183 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:47,183 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/A, priority=13, startTime=1733206546735; duration=0sec 2024-12-03T06:15:47,183 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:47,183 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:A 2024-12-03T06:15:47,192 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/789448c7c153409c99239caee331787a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/789448c7c153409c99239caee331787a 2024-12-03T06:15:47,197 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/C of b3fa0bec8819326450abec32a5991191 into 789448c7c153409c99239caee331787a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:47,197 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:47,197 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/C, priority=13, startTime=1733206546735; duration=0sec 2024-12-03T06:15:47,197 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:47,198 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:C 2024-12-03T06:15:47,269 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/55b4bd0e16d74a3e9f8201f4addffc0b 2024-12-03T06:15:47,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/415771db5e634153a08fab3362e57227 is 50, key is test_row_0/B:col10/1733206545873/Put/seqid=0 2024-12-03T06:15:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742402_1578 (size=12001) 2024-12-03T06:15:47,280 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/415771db5e634153a08fab3362e57227 2024-12-03T06:15:47,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9c965213b7734f34bc2ed5ab7c4e9374 is 50, key is test_row_0/C:col10/1733206545873/Put/seqid=0 2024-12-03T06:15:47,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742403_1579 (size=12001) 2024-12-03T06:15:47,301 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9c965213b7734f34bc2ed5ab7c4e9374 2024-12-03T06:15:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/55b4bd0e16d74a3e9f8201f4addffc0b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b 2024-12-03T06:15:47,312 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b, entries=150, sequenceid=84, filesize=30.2 K 2024-12-03T06:15:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/415771db5e634153a08fab3362e57227 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227 2024-12-03T06:15:47,316 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T06:15:47,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9c965213b7734f34bc2ed5ab7c4e9374 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374 2024-12-03T06:15:47,322 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T06:15:47,323 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b3fa0bec8819326450abec32a5991191 in 487ms, sequenceid=84, compaction requested=false 2024-12-03T06:15:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-03T06:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-03T06:15:47,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-03T06:15:47,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-12-03T06:15:47,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5640 sec 2024-12-03T06:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:47,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:47,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:47,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e757a00eaf474690ae21d6619b57acc7_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742404_1580 (size=17034) 2024-12-03T06:15:47,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206607370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206607373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206607374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206607480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206607489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206607489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206607690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206607700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206607700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:47,768 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:47,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e757a00eaf474690ae21d6619b57acc7_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e757a00eaf474690ae21d6619b57acc7_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:47,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/a4d98b21dc064114b137a95136802ad0, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:47,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/a4d98b21dc064114b137a95136802ad0 is 175, key is test_row_0/A:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:47,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742405_1581 (size=48139) 2024-12-03T06:15:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T06:15:47,920 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:47,921 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-03T06:15:47,923 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:47,923 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:47,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206607989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:47,991 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206607999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206608000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206608010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206608010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:48,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/a4d98b21dc064114b137a95136802ad0 2024-12-03T06:15:48,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/5503d068dbbe45da8209ac86fea843f0 is 50, key is test_row_0/B:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:48,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742406_1582 (size=12001) 2024-12-03T06:15:48,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:48,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206608510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206608510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206608519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206608519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:48,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:48,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/5503d068dbbe45da8209ac86fea843f0 2024-12-03T06:15:48,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/a608212238fb49caa6d866133a3887a4 is 50, key is test_row_0/C:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:48,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742407_1583 (size=12001) 2024-12-03T06:15:48,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:48,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:48,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/a608212238fb49caa6d866133a3887a4 2024-12-03T06:15:48,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/a4d98b21dc064114b137a95136802ad0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0 2024-12-03T06:15:48,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0, entries=250, sequenceid=102, filesize=47.0 K 2024-12-03T06:15:48,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/5503d068dbbe45da8209ac86fea843f0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0 2024-12-03T06:15:49,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0, entries=150, sequenceid=102, filesize=11.7 K 2024-12-03T06:15:49,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/a608212238fb49caa6d866133a3887a4 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4 2024-12-03T06:15:49,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4, entries=150, sequenceid=102, filesize=11.7 K 2024-12-03T06:15:49,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b3fa0bec8819326450abec32a5991191 in 1653ms, sequenceid=102, compaction requested=true 2024-12-03T06:15:49,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:49,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:49,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:49,004 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:49,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:49,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:49,004 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:49,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:49,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:49,005 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:49,005 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:49,005 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/B is initiating minor compaction (all files) 2024-12-03T06:15:49,005 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/A is initiating minor compaction (all files) 2024-12-03T06:15:49,005 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/A in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:49,005 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/B in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:49,005 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=107.6 K 2024-12-03T06:15:49,005 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c23e7464d0f34e1ca447d049f25ad47f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.3 K 2024-12-03T06:15:49,005 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:49,005 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0] 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c23e7464d0f34e1ca447d049f25ad47f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae42da0b8114e45a23ab415b03ebe15, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 415771db5e634153a08fab3362e57227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733206545873 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b4bd0e16d74a3e9f8201f4addffc0b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733206545873 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5503d068dbbe45da8209ac86fea843f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:49,006 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d98b21dc064114b137a95136802ad0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:49,010 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:49,011 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:49,012 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/b2ff5042d13e45b4a9b8da8c32462e03 is 50, key is test_row_0/B:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:49,014 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203dce64a1adf344638b10f467a1eced852_b3fa0bec8819326450abec32a5991191 store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:49,016 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203dce64a1adf344638b10f467a1eced852_b3fa0bec8819326450abec32a5991191, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:49,016 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dce64a1adf344638b10f467a1eced852_b3fa0bec8819326450abec32a5991191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:49,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742409_1585 (size=4469) 2024-12-03T06:15:49,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742408_1584 (size=12207) 2024-12-03T06:15:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:49,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T06:15:49,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:49,143 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:49,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035fd1933ac88b4e0b99824a5d8c24f543_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206547373/Put/seqid=0 2024-12-03T06:15:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742410_1586 (size=12154) 2024-12-03T06:15:49,429 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:49,429 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/06614d32d3b440b4b769db8cf6824990 is 175, key is test_row_0/A:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:49,432 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/b2ff5042d13e45b4a9b8da8c32462e03 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2ff5042d13e45b4a9b8da8c32462e03 2024-12-03T06:15:49,436 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/B of b3fa0bec8819326450abec32a5991191 into b2ff5042d13e45b4a9b8da8c32462e03(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:49,436 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:49,436 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/B, priority=13, startTime=1733206549004; duration=0sec 2024-12-03T06:15:49,436 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:49,437 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:B 2024-12-03T06:15:49,437 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:49,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:49,438 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/C is initiating minor compaction (all files) 2024-12-03T06:15:49,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/C in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:49,438 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/789448c7c153409c99239caee331787a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.3 K 2024-12-03T06:15:49,439 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 789448c7c153409c99239caee331787a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733206545850 2024-12-03T06:15:49,439 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c965213b7734f34bc2ed5ab7c4e9374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733206545873 2024-12-03T06:15:49,439 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a608212238fb49caa6d866133a3887a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:49,451 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:49,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742411_1587 (size=31161) 2024-12-03T06:15:49,451 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/71926db6685544fd919535f6bf576682 is 50, key is test_row_0/C:col10/1733206547027/Put/seqid=0 2024-12-03T06:15:49,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742412_1588 (size=12207) 2024-12-03T06:15:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:49,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206609543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206609544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206609544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206609546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:49,560 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035fd1933ac88b4e0b99824a5d8c24f543_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fd1933ac88b4e0b99824a5d8c24f543_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/e1c78fc25ec84659b27275be7ba8d382, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/e1c78fc25ec84659b27275be7ba8d382 is 175, key is test_row_0/A:col10/1733206547373/Put/seqid=0 2024-12-03T06:15:49,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742413_1589 (size=30955) 2024-12-03T06:15:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206609650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206609650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206609650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206609659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,854 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/06614d32d3b440b4b769db8cf6824990 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990 2024-12-03T06:15:49,857 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/A of b3fa0bec8819326450abec32a5991191 into 06614d32d3b440b4b769db8cf6824990(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:49,858 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:49,858 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/A, priority=13, startTime=1733206549004; duration=0sec 2024-12-03T06:15:49,858 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:49,858 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:A 2024-12-03T06:15:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206609860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206609860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206609860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,865 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/71926db6685544fd919535f6bf576682 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/71926db6685544fd919535f6bf576682 2024-12-03T06:15:49,868 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/C of b3fa0bec8819326450abec32a5991191 into 71926db6685544fd919535f6bf576682(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:49,868 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:49,868 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/C, priority=13, startTime=1733206549004; duration=0sec 2024-12-03T06:15:49,868 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:49,868 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:C 2024-12-03T06:15:49,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206609869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:49,964 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/e1c78fc25ec84659b27275be7ba8d382 2024-12-03T06:15:49,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c77a0bc3313c4c06a5857a7c77db431e is 50, key is test_row_0/B:col10/1733206547373/Put/seqid=0 2024-12-03T06:15:49,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742414_1590 (size=12001) 2024-12-03T06:15:49,974 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c77a0bc3313c4c06a5857a7c77db431e 2024-12-03T06:15:49,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/0ca78ed52d024d5dbcd8d9b1d79c3240 is 50, key is test_row_0/C:col10/1733206547373/Put/seqid=0 2024-12-03T06:15:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742415_1591 (size=12001) 2024-12-03T06:15:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206610169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206610169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206610169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206610179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,388 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/0ca78ed52d024d5dbcd8d9b1d79c3240 2024-12-03T06:15:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/e1c78fc25ec84659b27275be7ba8d382 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382 2024-12-03T06:15:50,394 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382, entries=150, sequenceid=118, filesize=30.2 K 2024-12-03T06:15:50,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c77a0bc3313c4c06a5857a7c77db431e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e 2024-12-03T06:15:50,397 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T06:15:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/0ca78ed52d024d5dbcd8d9b1d79c3240 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240 2024-12-03T06:15:50,400 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T06:15:50,401 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b3fa0bec8819326450abec32a5991191 in 1258ms, sequenceid=118, compaction requested=false 2024-12-03T06:15:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-03T06:15:50,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-03T06:15:50,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-03T06:15:50,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4790 sec 2024-12-03T06:15:50,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.4830 sec 2024-12-03T06:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:50,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:15:50,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:50,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2da6f6c460b49029ffc7133e2f3ec7e_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:50,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742416_1592 (size=12304) 2024-12-03T06:15:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206610696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206610697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206610697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206610698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206610809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206610809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206610809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206610809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206611019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206611019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206611020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206611020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:51,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2da6f6c460b49029ffc7133e2f3ec7e_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2da6f6c460b49029ffc7133e2f3ec7e_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:51,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/bdb2e742f063494494ecdd69364c1660, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:51,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/bdb2e742f063494494ecdd69364c1660 is 175, key is test_row_0/A:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:51,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742417_1593 (size=31105) 2024-12-03T06:15:51,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/bdb2e742f063494494ecdd69364c1660 2024-12-03T06:15:51,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/7aa390823e2a47a0970d89df3e1f8f2e is 50, key is test_row_0/B:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:51,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742418_1594 (size=12151) 2024-12-03T06:15:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206611335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206611335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/7aa390823e2a47a0970d89df3e1f8f2e 2024-12-03T06:15:51,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/87a8449b6fb8438e8509864287c81791 is 50, key is test_row_0/C:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:51,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742419_1595 (size=12151) 2024-12-03T06:15:51,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206611850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206611850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206611850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206611850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:51,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/87a8449b6fb8438e8509864287c81791 2024-12-03T06:15:51,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/bdb2e742f063494494ecdd69364c1660 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660 2024-12-03T06:15:51,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660, entries=150, sequenceid=140, filesize=30.4 K 2024-12-03T06:15:51,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/7aa390823e2a47a0970d89df3e1f8f2e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e 2024-12-03T06:15:51,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e, entries=150, sequenceid=140, filesize=11.9 K 2024-12-03T06:15:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/87a8449b6fb8438e8509864287c81791 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791 2024-12-03T06:15:51,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791, entries=150, sequenceid=140, filesize=11.9 K 2024-12-03T06:15:51,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b3fa0bec8819326450abec32a5991191 in 1258ms, sequenceid=140, compaction requested=true 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:51,941 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:51,941 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:51,941 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:51,941 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/A is initiating minor compaction (all files) 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/B is initiating minor compaction (all files) 2024-12-03T06:15:51,942 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/B in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:51,942 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/A in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:51,942 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2ff5042d13e45b4a9b8da8c32462e03, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.5 K 2024-12-03T06:15:51,942 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=91.0 K 2024-12-03T06:15:51,942 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660] 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ff5042d13e45b4a9b8da8c32462e03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06614d32d3b440b4b769db8cf6824990, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c77a0bc3313c4c06a5857a7c77db431e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733206547369 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c78fc25ec84659b27275be7ba8d382, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733206547369 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa390823e2a47a0970d89df3e1f8f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:51,942 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb2e742f063494494ecdd69364c1660, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:51,946 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:51,948 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:51,948 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/ee52b882659043909b2eeb2f7e804cbe is 50, key is test_row_0/B:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:51,950 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033f1d6a4fbebf45749cd944c22273a12a_b3fa0bec8819326450abec32a5991191 store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:51,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742420_1596 (size=12459) 2024-12-03T06:15:51,952 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033f1d6a4fbebf45749cd944c22273a12a_b3fa0bec8819326450abec32a5991191, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:51,952 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033f1d6a4fbebf45749cd944c22273a12a_b3fa0bec8819326450abec32a5991191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:51,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742421_1597 (size=4469) 2024-12-03T06:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:52,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:52,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a6a6c907fe8549ca9c28cc1c99f4db17_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206550697/Put/seqid=0 2024-12-03T06:15:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742422_1598 (size=14794) 2024-12-03T06:15:52,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206612066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T06:15:52,090 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:52,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-03T06:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:52,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:52,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:52,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206612179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:52,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,356 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:52,356 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/ee52b882659043909b2eeb2f7e804cbe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ee52b882659043909b2eeb2f7e804cbe 2024-12-03T06:15:52,356 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/834c9c192e894474acdb7733cf203d3a is 175, key is test_row_0/A:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:52,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742423_1599 (size=31413) 2024-12-03T06:15:52,360 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/B of b3fa0bec8819326450abec32a5991191 into ee52b882659043909b2eeb2f7e804cbe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:52,360 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:52,360 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/B, priority=13, startTime=1733206551941; duration=0sec 2024-12-03T06:15:52,360 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:52,360 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:B 2024-12-03T06:15:52,361 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T06:15:52,361 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T06:15:52,361 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/C is initiating minor compaction (all files) 2024-12-03T06:15:52,361 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/C in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,361 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/71926db6685544fd919535f6bf576682, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=35.5 K 2024-12-03T06:15:52,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 71926db6685544fd919535f6bf576682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733206547027 2024-12-03T06:15:52,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca78ed52d024d5dbcd8d9b1d79c3240, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733206547369 2024-12-03T06:15:52,362 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a8449b6fb8438e8509864287c81791, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:52,368 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:52,368 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/a1230bc3eb63448892efb6b2139d73c8 is 50, key is test_row_0/C:col10/1733206550682/Put/seqid=0 2024-12-03T06:15:52,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742424_1600 (size=12459) 2024-12-03T06:15:52,377 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/a1230bc3eb63448892efb6b2139d73c8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a1230bc3eb63448892efb6b2139d73c8 2024-12-03T06:15:52,380 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/C of b3fa0bec8819326450abec32a5991191 into a1230bc3eb63448892efb6b2139d73c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:52,380 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:52,380 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/C, priority=13, startTime=1733206551941; duration=0sec 2024-12-03T06:15:52,380 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:52,380 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:C 2024-12-03T06:15:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206612389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:52,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:52,414 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a6a6c907fe8549ca9c28cc1c99f4db17_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a6a6c907fe8549ca9c28cc1c99f4db17_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:52,414 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/49cb691e03704ab5af7d31317f51321a, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:52,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/49cb691e03704ab5af7d31317f51321a is 175, key is test_row_0/A:col10/1733206550697/Put/seqid=0 2024-12-03T06:15:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742425_1601 (size=39749) 2024-12-03T06:15:52,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:52,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:52,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:52,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206612699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:52,764 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/834c9c192e894474acdb7733cf203d3a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a 2024-12-03T06:15:52,768 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3fa0bec8819326450abec32a5991191/A of b3fa0bec8819326450abec32a5991191 into 834c9c192e894474acdb7733cf203d3a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:52,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:52,768 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/A, priority=13, startTime=1733206551941; duration=0sec 2024-12-03T06:15:52,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:52,768 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:A 2024-12-03T06:15:52,818 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/49cb691e03704ab5af7d31317f51321a 2024-12-03T06:15:52,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c41158682ba44fadb34b6e17a342448c is 50, key is test_row_0/B:col10/1733206550697/Put/seqid=0 2024-12-03T06:15:52,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742426_1602 (size=12151) 2024-12-03T06:15:52,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c41158682ba44fadb34b6e17a342448c 2024-12-03T06:15:52,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/171d9b41338a40b997ff8dbdfff52590 is 50, key is test_row_0/C:col10/1733206550697/Put/seqid=0 2024-12-03T06:15:52,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:52,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:15:52,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206612860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206612860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206612860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,863 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:52,863 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,863 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742427_1603 (size=12151) 2024-12-03T06:15:52,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/171d9b41338a40b997ff8dbdfff52590 2024-12-03T06:15:52,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/49cb691e03704ab5af7d31317f51321a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a 2024-12-03T06:15:52,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206612869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:52,870 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:52,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a, entries=200, sequenceid=154, filesize=38.8 K 2024-12-03T06:15:52,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/c41158682ba44fadb34b6e17a342448c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c 2024-12-03T06:15:52,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c, entries=150, sequenceid=154, filesize=11.9 K 2024-12-03T06:15:52,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/171d9b41338a40b997ff8dbdfff52590 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590 2024-12-03T06:15:52,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590, entries=150, sequenceid=154, filesize=11.9 K 2024-12-03T06:15:52,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b3fa0bec8819326450abec32a5991191 in 879ms, sequenceid=154, compaction requested=false 2024-12-03T06:15:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:53,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:53,007 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:53,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f44d5fb8dcc4468e9c3139933b98b08a_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206552051/Put/seqid=0 2024-12-03T06:15:53,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742428_1604 (size=12304) 2024-12-03T06:15:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:53,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:53,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206613242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:53,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206613349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:53,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:53,435 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f44d5fb8dcc4468e9c3139933b98b08a_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f44d5fb8dcc4468e9c3139933b98b08a_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/0775d3b2fb9a428fbee04d73456500ca, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:53,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/0775d3b2fb9a428fbee04d73456500ca is 175, key is test_row_0/A:col10/1733206552051/Put/seqid=0 2024-12-03T06:15:53,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742429_1605 (size=31105) 2024-12-03T06:15:53,440 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/0775d3b2fb9a428fbee04d73456500ca 2024-12-03T06:15:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/6c73ec0cb7244e1c81d3eaff68ccfdfe is 50, key is test_row_0/B:col10/1733206552051/Put/seqid=0 2024-12-03T06:15:53,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742430_1606 (size=12151) 2024-12-03T06:15:53,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206613559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:53,850 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/6c73ec0cb7244e1c81d3eaff68ccfdfe 2024-12-03T06:15:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d28177d96f79444aac706b97c9a49e82 is 50, key is test_row_0/C:col10/1733206552051/Put/seqid=0 2024-12-03T06:15:53,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742431_1607 (size=12151) 2024-12-03T06:15:53,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206613870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:53,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:54,259 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d28177d96f79444aac706b97c9a49e82 2024-12-03T06:15:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/0775d3b2fb9a428fbee04d73456500ca as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca 2024-12-03T06:15:54,265 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca, entries=150, sequenceid=175, filesize=30.4 K 2024-12-03T06:15:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/6c73ec0cb7244e1c81d3eaff68ccfdfe as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe 2024-12-03T06:15:54,268 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe, entries=150, sequenceid=175, filesize=11.9 K 2024-12-03T06:15:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d28177d96f79444aac706b97c9a49e82 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82 2024-12-03T06:15:54,271 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82, entries=150, sequenceid=175, filesize=11.9 K 2024-12-03T06:15:54,272 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b3fa0bec8819326450abec32a5991191 in 1264ms, sequenceid=175, compaction requested=true 2024-12-03T06:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-03T06:15:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-03T06:15:54,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-03T06:15:54,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-12-03T06:15:54,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.1840 sec 2024-12-03T06:15:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:54,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:15:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:54,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:54,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:54,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:54,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:54,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d2148d48901c4f198787140c3af7fe08_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:54,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742432_1608 (size=12304) 2024-12-03T06:15:54,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206614434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206614540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206614749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:54,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d2148d48901c4f198787140c3af7fe08_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2148d48901c4f198787140c3af7fe08_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:54,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/b13ff4fc83d24a9ebc089cb854199463, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:54,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/b13ff4fc83d24a9ebc089cb854199463 is 175, key is test_row_0/A:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:54,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742433_1609 (size=31105) 2024-12-03T06:15:54,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206614869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,871 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206614880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,881 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206614890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206614890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:54,891 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:54,891 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:55,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206615059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:55,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/b13ff4fc83d24a9ebc089cb854199463 2024-12-03T06:15:55,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9b7d590ac04f4424bd2df48ed0f746e6 is 50, key is test_row_0/B:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742434_1610 (size=12151) 2024-12-03T06:15:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206615569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:55,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9b7d590ac04f4424bd2df48ed0f746e6 2024-12-03T06:15:55,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/b2310174ffde4758904dccc2f8e0941c is 50, key is test_row_0/C:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742435_1611 (size=12151) 2024-12-03T06:15:56,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/b2310174ffde4758904dccc2f8e0941c 2024-12-03T06:15:56,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/b13ff4fc83d24a9ebc089cb854199463 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463 2024-12-03T06:15:56,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463, entries=150, sequenceid=190, filesize=30.4 K 2024-12-03T06:15:56,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9b7d590ac04f4424bd2df48ed0f746e6 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6 2024-12-03T06:15:56,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T06:15:56,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/b2310174ffde4758904dccc2f8e0941c as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c 2024-12-03T06:15:56,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T06:15:56,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b3fa0bec8819326450abec32a5991191 in 1658ms, sequenceid=190, compaction requested=true 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:56,040 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:15:56,040 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:15:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:56,041 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:15:56,041 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:15:56,042 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/A is initiating minor compaction (all files) 2024-12-03T06:15:56,042 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/B is initiating minor compaction (all files) 2024-12-03T06:15:56,042 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/B in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:56,042 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/A in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:56,042 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ee52b882659043909b2eeb2f7e804cbe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=47.8 K 2024-12-03T06:15:56,042 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=130.2 K 2024-12-03T06:15:56,042 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:56,042 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463] 2024-12-03T06:15:56,043 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834c9c192e894474acdb7733cf203d3a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:56,043 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting ee52b882659043909b2eeb2f7e804cbe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:56,043 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cb691e03704ab5af7d31317f51321a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733206550695 2024-12-03T06:15:56,043 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting c41158682ba44fadb34b6e17a342448c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733206550695 2024-12-03T06:15:56,043 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0775d3b2fb9a428fbee04d73456500ca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733206552042 2024-12-03T06:15:56,044 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c73ec0cb7244e1c81d3eaff68ccfdfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733206552042 2024-12-03T06:15:56,044 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b13ff4fc83d24a9ebc089cb854199463, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:15:56,046 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7d590ac04f4424bd2df48ed0f746e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:15:56,058 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:56,059 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:56,060 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/10b8792f208943998f22199b9d550fb7 is 50, key is test_row_0/B:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:56,060 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c6cf9a0bf9dc4352b69aa3921d90e398_b3fa0bec8819326450abec32a5991191 store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:56,063 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c6cf9a0bf9dc4352b69aa3921d90e398_b3fa0bec8819326450abec32a5991191, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:56,064 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c6cf9a0bf9dc4352b69aa3921d90e398_b3fa0bec8819326450abec32a5991191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:56,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742436_1612 (size=12595) 2024-12-03T06:15:56,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742437_1613 (size=4469) 2024-12-03T06:15:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T06:15:56,260 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:56,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-12-03T06:15:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:56,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:56,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:56,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:56,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-03T06:15:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:56,414 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:56,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d29618e7212f4b2ea9ac8cef7a0613c6_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206554433/Put/seqid=0 2024-12-03T06:15:56,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742438_1614 (size=12304) 2024-12-03T06:15:56,475 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/10b8792f208943998f22199b9d550fb7 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/10b8792f208943998f22199b9d550fb7 2024-12-03T06:15:56,479 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3fa0bec8819326450abec32a5991191/B of b3fa0bec8819326450abec32a5991191 into 10b8792f208943998f22199b9d550fb7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:56,479 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:56,479 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/B, priority=12, startTime=1733206556040; duration=0sec 2024-12-03T06:15:56,479 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:15:56,479 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:B 2024-12-03T06:15:56,479 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T06:15:56,480 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T06:15:56,480 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/C is initiating minor compaction (all files) 2024-12-03T06:15:56,480 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/C in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:56,480 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a1230bc3eb63448892efb6b2139d73c8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=47.8 K 2024-12-03T06:15:56,480 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting a1230bc3eb63448892efb6b2139d73c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733206549543 2024-12-03T06:15:56,480 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 171d9b41338a40b997ff8dbdfff52590, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733206550695 2024-12-03T06:15:56,480 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d28177d96f79444aac706b97c9a49e82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733206552042 2024-12-03T06:15:56,481 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b2310174ffde4758904dccc2f8e0941c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:15:56,487 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:56,487 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d0c00708331b4394b0f295f5be2632aa is 50, key is test_row_0/C:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:56,490 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:15:56,491 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/cdafffb32ee84b879e01371aaaccf6ba is 175, key is test_row_0/A:col10/1733206553225/Put/seqid=0 2024-12-03T06:15:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742439_1615 (size=12595) 2024-12-03T06:15:56,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742440_1616 (size=31549) 2024-12-03T06:15:56,496 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d0c00708331b4394b0f295f5be2632aa as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d0c00708331b4394b0f295f5be2632aa 2024-12-03T06:15:56,498 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/cdafffb32ee84b879e01371aaaccf6ba as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba 2024-12-03T06:15:56,500 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3fa0bec8819326450abec32a5991191/C of b3fa0bec8819326450abec32a5991191 into d0c00708331b4394b0f295f5be2632aa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:56,500 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:56,500 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/C, priority=12, startTime=1733206556040; duration=0sec 2024-12-03T06:15:56,500 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:56,500 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:C 2024-12-03T06:15:56,502 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3fa0bec8819326450abec32a5991191/A of b3fa0bec8819326450abec32a5991191 into cdafffb32ee84b879e01371aaaccf6ba(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:15:56,502 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:56,502 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/A, priority=12, startTime=1733206556040; duration=0sec 2024-12-03T06:15:56,502 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:15:56,502 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:A 2024-12-03T06:15:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:56,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206616628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:56,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206616740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:56,831 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d29618e7212f4b2ea9ac8cef7a0613c6_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d29618e7212f4b2ea9ac8cef7a0613c6_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/dc2bc39d39e34326976bfa082ef386f1, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:56,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/dc2bc39d39e34326976bfa082ef386f1 is 175, key is test_row_0/A:col10/1733206554433/Put/seqid=0 2024-12-03T06:15:56,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742441_1617 (size=31105) 2024-12-03T06:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:56,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206616949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:57,243 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/dc2bc39d39e34326976bfa082ef386f1 2024-12-03T06:15:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 is 50, key is test_row_0/B:col10/1733206554433/Put/seqid=0 2024-12-03T06:15:57,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742442_1618 (size=12151) 2024-12-03T06:15:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206617259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:57,653 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 2024-12-03T06:15:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/5710ba21756041cbb42f46ada3732abb is 50, key is test_row_0/C:col10/1733206554433/Put/seqid=0 2024-12-03T06:15:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742443_1619 (size=12151) 2024-12-03T06:15:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206617769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,071 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/5710ba21756041cbb42f46ada3732abb 2024-12-03T06:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/dc2bc39d39e34326976bfa082ef386f1 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1 2024-12-03T06:15:58,087 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1, entries=150, sequenceid=207, filesize=30.4 K 2024-12-03T06:15:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 2024-12-03T06:15:58,098 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T06:15:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/5710ba21756041cbb42f46ada3732abb as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb 2024-12-03T06:15:58,116 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T06:15:58,121 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b3fa0bec8819326450abec32a5991191 in 1707ms, sequenceid=207, compaction requested=false 2024-12-03T06:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-12-03T06:15:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-12-03T06:15:58,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-03T06:15:58,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-12-03T06:15:58,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.8670 sec 2024-12-03T06:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T06:15:58,410 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:58,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-12-03T06:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T06:15:58,413 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:58,414 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:58,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T06:15:58,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-03T06:15:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:58,567 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T06:15:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c7eaab600134f778cf5a2a927e7080f_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206556598/Put/seqid=0 2024-12-03T06:15:58,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742444_1620 (size=12304) 2024-12-03T06:15:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:58,595 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c7eaab600134f778cf5a2a927e7080f_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c7eaab600134f778cf5a2a927e7080f_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/945b35d907504ac7b00c54c02fd577b5, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/945b35d907504ac7b00c54c02fd577b5 is 175, key is test_row_0/A:col10/1733206556598/Put/seqid=0 2024-12-03T06:15:58,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742445_1621 (size=31105) 2024-12-03T06:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T06:15:58,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:15:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:58,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206618860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733206618879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,883 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34932 deadline: 1733206618911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,913 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34934 deadline: 1733206618919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,922 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_1' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34972 deadline: 1733206618940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,944 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 122cddc95ec8,35255,1733206381424 for 'test_row_0' in b3fa0bec8819326450abec32a5991191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T06:15:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:58,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206618970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:59,031 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/945b35d907504ac7b00c54c02fd577b5 2024-12-03T06:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T06:15:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9e061b264b914eada7969f24da8441b8 is 50, key is test_row_0/B:col10/1733206556598/Put/seqid=0 2024-12-03T06:15:59,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742446_1622 (size=12151) 2024-12-03T06:15:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206619180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:59,481 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9e061b264b914eada7969f24da8441b8 2024-12-03T06:15:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9e1e391c9fe24bf8b7252b64388a3a65 is 50, key is test_row_0/C:col10/1733206556598/Put/seqid=0 2024-12-03T06:15:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206619489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:15:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:15:59,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742447_1623 (size=12151) 2024-12-03T06:15:59,512 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9e1e391c9fe24bf8b7252b64388a3a65 2024-12-03T06:15:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/945b35d907504ac7b00c54c02fd577b5 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5 2024-12-03T06:15:59,527 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5, entries=150, sequenceid=225, filesize=30.4 K 2024-12-03T06:15:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9e061b264b914eada7969f24da8441b8 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8 2024-12-03T06:15:59,531 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8, entries=150, sequenceid=225, filesize=11.9 K 2024-12-03T06:15:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/9e1e391c9fe24bf8b7252b64388a3a65 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65 2024-12-03T06:15:59,535 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65, entries=150, sequenceid=225, filesize=11.9 K 2024-12-03T06:15:59,536 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b3fa0bec8819326450abec32a5991191 in 969ms, sequenceid=225, compaction requested=true 2024-12-03T06:15:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:15:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-12-03T06:15:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-12-03T06:15:59,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-03T06:15:59,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1230 sec 2024-12-03T06:15:59,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.1280 sec 2024-12-03T06:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T06:15:59,550 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:15:59,550 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-12-03T06:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T06:15:59,552 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:15:59,553 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:15:59,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:15:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T06:15:59,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-12-03T06:15:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:15:59,706 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:15:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:15:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037b8cb5e9d16649f8af9209e422f43b4c_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206558849/Put/seqid=0 2024-12-03T06:15:59,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742448_1624 (size=12304) 2024-12-03T06:15:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:15:59,738 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037b8cb5e9d16649f8af9209e422f43b4c_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037b8cb5e9d16649f8af9209e422f43b4c_b3fa0bec8819326450abec32a5991191 2024-12-03T06:15:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/32bdc8ce5e424f59ae8a154e53e8c35b, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:15:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/32bdc8ce5e424f59ae8a154e53e8c35b is 175, key is test_row_0/A:col10/1733206558849/Put/seqid=0 2024-12-03T06:15:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742449_1625 (size=31105) 2024-12-03T06:15:59,769 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T06:15:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T06:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:00,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:00,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206620067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:00,144 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/32bdc8ce5e424f59ae8a154e53e8c35b 2024-12-03T06:16:00,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/622e0e1776874a829944ec2c076fc9be is 50, key is test_row_0/B:col10/1733206558849/Put/seqid=0 2024-12-03T06:16:00,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742450_1626 (size=12151) 2024-12-03T06:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T06:16:00,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206620179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206620389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:00,556 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/622e0e1776874a829944ec2c076fc9be 2024-12-03T06:16:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7b2e7313edcf4d43aa6d954f291ef9ed is 50, key is test_row_0/C:col10/1733206558849/Put/seqid=0 2024-12-03T06:16:00,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742451_1627 (size=12151) 2024-12-03T06:16:00,572 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7b2e7313edcf4d43aa6d954f291ef9ed 2024-12-03T06:16:00,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/32bdc8ce5e424f59ae8a154e53e8c35b as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b 2024-12-03T06:16:00,583 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b, entries=150, sequenceid=242, filesize=30.4 K 2024-12-03T06:16:00,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/622e0e1776874a829944ec2c076fc9be as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be 2024-12-03T06:16:00,586 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be, entries=150, sequenceid=242, filesize=11.9 K 2024-12-03T06:16:00,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/7b2e7313edcf4d43aa6d954f291ef9ed as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed 2024-12-03T06:16:00,590 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed, entries=150, sequenceid=242, filesize=11.9 K 2024-12-03T06:16:00,591 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b3fa0bec8819326450abec32a5991191 in 886ms, sequenceid=242, compaction requested=true 2024-12-03T06:16:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-12-03T06:16:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-12-03T06:16:00,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-12-03T06:16:00,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-12-03T06:16:00,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.0450 sec 2024-12-03T06:16:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T06:16:00,690 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:16:00,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T06:16:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-12-03T06:16:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:00,692 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T06:16:00,692 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T06:16:00,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T06:16:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:00,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:16:00,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:16:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:16:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:16:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:00,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036d5c13bea08d4d4db68867878a644fbf_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:00,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742452_1628 (size=12354) 2024-12-03T06:16:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206620778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:00,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:00,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:00,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206620889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:00,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:00,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:01,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206621099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:01,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:16:01,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036d5c13bea08d4d4db68867878a644fbf_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d5c13bea08d4d4db68867878a644fbf_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:01,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/5c214225c5fe4165978264086b447a90, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:01,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/5c214225c5fe4165978264086b447a90 is 175, key is test_row_0/A:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:01,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742453_1629 (size=31155) 2024-12-03T06:16:01,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/5c214225c5fe4165978264086b447a90 2024-12-03T06:16:01,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/24b0b1d3b1ba4fe58310968abbdaea3a is 50, key is test_row_0/B:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742454_1630 (size=12201) 2024-12-03T06:16:01,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:01,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206621410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:01,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/24b0b1d3b1ba4fe58310968abbdaea3a 2024-12-03T06:16:01,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/8c1615a4f00b449097eb8509a7ff76cf is 50, key is test_row_0/C:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:01,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742455_1631 (size=12201) 2024-12-03T06:16:01,608 INFO [Thread-2500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2500. 2024-12-03T06:16:01,608 DEBUG [Thread-2500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:01,608 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,609 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-12-03T06:16:01,609 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,609 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:01,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:01,609 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,609 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:01,610 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-12-03T06:16:01,610 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:01,610 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,610 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:01,612 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-12-03T06:16:01,612 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:01,612 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,612 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:01,612 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-12-03T06:16:01,612 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:01,612 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,612 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:01,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:01,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34918 deadline: 1733206621919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 , the old value is region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T06:16:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., hostname=122cddc95ec8,35255,1733206381424, seqNum=5 because the exception is null or not the one we care about 2024-12-03T06:16:01,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:16:01,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/8c1615a4f00b449097eb8509a7ff76cf 2024-12-03T06:16:01,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/5c214225c5fe4165978264086b447a90 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90 2024-12-03T06:16:01,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90, entries=150, sequenceid=258, filesize=30.4 K 2024-12-03T06:16:01,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/24b0b1d3b1ba4fe58310968abbdaea3a as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a 2024-12-03T06:16:01,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T06:16:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/8c1615a4f00b449097eb8509a7ff76cf as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf 2024-12-03T06:16:01,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T06:16:01,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b3fa0bec8819326450abec32a5991191 in 1268ms, sequenceid=258, compaction requested=true 2024-12-03T06:16:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T06:16:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:16:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T06:16:01,971 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:16:01,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:16:01,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3fa0bec8819326450abec32a5991191:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T06:16:01,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:16:01,972 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:16:01,972 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156019 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:16:01,972 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61249 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/B is initiating minor compaction (all files) 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/A is initiating minor compaction (all files) 2024-12-03T06:16:01,973 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/A in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,973 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/B in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,973 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/10b8792f208943998f22199b9d550fb7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=59.8 K 2024-12-03T06:16:01,973 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=152.4 K 2024-12-03T06:16:01,973 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. files: [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90] 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b8792f208943998f22199b9d550fb7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdafffb32ee84b879e01371aaaccf6ba, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting b2308ed1f2fb49c0bb0f7a9cd9413d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733206554425 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2bc39d39e34326976bfa082ef386f1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733206554425 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e061b264b914eada7969f24da8441b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733206556598 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 945b35d907504ac7b00c54c02fd577b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733206556598 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 622e0e1776874a829944ec2c076fc9be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733206558839 2024-12-03T06:16:01,973 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bdc8ce5e424f59ae8a154e53e8c35b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733206558839 2024-12-03T06:16:01,974 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b0b1d3b1ba4fe58310968abbdaea3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733206560040 2024-12-03T06:16:01,974 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c214225c5fe4165978264086b447a90, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733206560040 2024-12-03T06:16:01,979 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:01,980 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#B#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:16:01,980 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/1a32b9c4af3d4c7bb3365a11ab8d2b13 is 50, key is test_row_0/B:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:01,981 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203288f6012031b4d4a988c881e7061e5cf_b3fa0bec8819326450abec32a5991191 store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:01,984 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203288f6012031b4d4a988c881e7061e5cf_b3fa0bec8819326450abec32a5991191, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:01,984 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203288f6012031b4d4a988c881e7061e5cf_b3fa0bec8819326450abec32a5991191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:01,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742456_1632 (size=12815) 2024-12-03T06:16:01,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742457_1633 (size=4469) 2024-12-03T06:16:01,988 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#A#compaction#535 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:16:01,989 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/4161be093f70463aafe43549ff1fab4e is 175, key is test_row_0/A:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:01,990 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/1a32b9c4af3d4c7bb3365a11ab8d2b13 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/1a32b9c4af3d4c7bb3365a11ab8d2b13 2024-12-03T06:16:01,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742458_1634 (size=31769) 2024-12-03T06:16:01,993 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b3fa0bec8819326450abec32a5991191/B of b3fa0bec8819326450abec32a5991191 into 1a32b9c4af3d4c7bb3365a11ab8d2b13(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:16:01,993 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:01,993 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/B, priority=11, startTime=1733206561971; duration=0sec 2024-12-03T06:16:01,993 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T06:16:01,993 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:B 2024-12-03T06:16:01,993 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T06:16:01,994 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61249 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T06:16:01,994 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1541): b3fa0bec8819326450abec32a5991191/C is initiating minor compaction (all files) 2024-12-03T06:16:01,994 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3fa0bec8819326450abec32a5991191/C in TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:01,994 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d0c00708331b4394b0f295f5be2632aa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf] into tmpdir=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp, totalSize=59.8 K 2024-12-03T06:16:01,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c00708331b4394b0f295f5be2632aa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733206553225 2024-12-03T06:16:01,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5710ba21756041cbb42f46ada3732abb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733206554425 2024-12-03T06:16:01,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1e391c9fe24bf8b7252b64388a3a65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733206556598 2024-12-03T06:16:01,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2e7313edcf4d43aa6d954f291ef9ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733206558839 2024-12-03T06:16:01,995 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c1615a4f00b449097eb8509a7ff76cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733206560040 2024-12-03T06:16:02,002 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3fa0bec8819326450abec32a5991191#C#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T06:16:02,003 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d10dc4903781473ca8154a23a4729be9 is 50, key is test_row_0/C:col10/1733206560059/Put/seqid=0 2024-12-03T06:16:02,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742459_1635 (size=12815) 2024-12-03T06:16:02,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:02,088 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e67539f945194faaa5eea0c5378d8b5c_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206560769/Put/seqid=0 2024-12-03T06:16:02,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742460_1636 (size=12454) 2024-12-03T06:16:02,395 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/4161be093f70463aafe43549ff1fab4e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/4161be093f70463aafe43549ff1fab4e 2024-12-03T06:16:02,398 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b3fa0bec8819326450abec32a5991191/A of b3fa0bec8819326450abec32a5991191 into 4161be093f70463aafe43549ff1fab4e(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:16:02,398 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:02,398 INFO [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/A, priority=11, startTime=1733206561971; duration=0sec 2024-12-03T06:16:02,398 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:16:02,398 DEBUG [RS:0;122cddc95ec8:35255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:A 2024-12-03T06:16:02,409 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/d10dc4903781473ca8154a23a4729be9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d10dc4903781473ca8154a23a4729be9 2024-12-03T06:16:02,413 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b3fa0bec8819326450abec32a5991191/C of b3fa0bec8819326450abec32a5991191 into d10dc4903781473ca8154a23a4729be9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T06:16:02,413 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:02,413 INFO [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., storeName=b3fa0bec8819326450abec32a5991191/C, priority=11, startTime=1733206561972; duration=0sec 2024-12-03T06:16:02,413 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T06:16:02,413 DEBUG [RS:0;122cddc95ec8:35255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3fa0bec8819326450abec32a5991191:C 2024-12-03T06:16:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:16:02,499 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e67539f945194faaa5eea0c5378d8b5c_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e67539f945194faaa5eea0c5378d8b5c_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/1167148d4f6b4022bf3ffb7a0df5dcef, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:02,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/1167148d4f6b4022bf3ffb7a0df5dcef is 175, key is test_row_0/A:col10/1733206560769/Put/seqid=0 2024-12-03T06:16:02,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742461_1637 (size=31255) 2024-12-03T06:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:02,904 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/1167148d4f6b4022bf3ffb7a0df5dcef 2024-12-03T06:16:02,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/ad765e0b57074075a7f133b0bcae447f is 50, key is test_row_0/B:col10/1733206560769/Put/seqid=0 2024-12-03T06:16:02,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742462_1638 (size=12301) 2024-12-03T06:16:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35255 {}] regionserver.HRegion(8855): Flush requested on b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:02,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. as already flushing 2024-12-03T06:16:02,931 INFO [Thread-2485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2485. 2024-12-03T06:16:02,931 DEBUG [Thread-2485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:02,931 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:02,931 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:02,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:03,321 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/ad765e0b57074075a7f133b0bcae447f 2024-12-03T06:16:03,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/603eecd42c5a43dab555c466e9ca1d8d is 50, key is test_row_0/C:col10/1733206560769/Put/seqid=0 2024-12-03T06:16:03,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742463_1639 (size=12301) 2024-12-03T06:16:03,729 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/603eecd42c5a43dab555c466e9ca1d8d 2024-12-03T06:16:03,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/1167148d4f6b4022bf3ffb7a0df5dcef as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/1167148d4f6b4022bf3ffb7a0df5dcef 2024-12-03T06:16:03,736 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/1167148d4f6b4022bf3ffb7a0df5dcef, entries=150, sequenceid=275, filesize=30.5 K 2024-12-03T06:16:03,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/ad765e0b57074075a7f133b0bcae447f as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ad765e0b57074075a7f133b0bcae447f 2024-12-03T06:16:03,739 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ad765e0b57074075a7f133b0bcae447f, entries=150, sequenceid=275, filesize=12.0 K 2024-12-03T06:16:03,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/603eecd42c5a43dab555c466e9ca1d8d as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/603eecd42c5a43dab555c466e9ca1d8d 2024-12-03T06:16:03,743 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/603eecd42c5a43dab555c466e9ca1d8d, entries=150, sequenceid=275, filesize=12.0 K 2024-12-03T06:16:03,744 INFO [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for b3fa0bec8819326450abec32a5991191 in 1656ms, sequenceid=275, compaction requested=false 2024-12-03T06:16:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for b3fa0bec8819326450abec32a5991191: 2024-12-03T06:16:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-12-03T06:16:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-12-03T06:16:03,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-12-03T06:16:03,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0530 sec 2024-12-03T06:16:03,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 3.0580 sec 2024-12-03T06:16:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T06:16:04,870 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:16:08,891 INFO [Thread-2483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2483. 2024-12-03T06:16:08,891 DEBUG [Thread-2483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:08,891 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:08,891 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:08,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:08,961 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-12-03T06:16:08,961 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:08,961 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:08,961 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:08,961 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:09,001 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-12-03T06:16:09,001 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:09,001 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,001 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:09,031 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-12-03T06:16:09,031 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T06:16:09,031 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,031 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3947 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3867 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3882 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3938 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3855 2024-12-03T06:16:09,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T06:16:09,032 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:16:09,032 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:16:09,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:09,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T06:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T06:16:09,036 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206569036"}]},"ts":"1733206569036"} 2024-12-03T06:16:09,037 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T06:16:09,037 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T06:16:09,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T06:16:09,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, UNASSIGN}] 2024-12-03T06:16:09,040 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, UNASSIGN 2024-12-03T06:16:09,040 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=CLOSING, regionLocation=122cddc95ec8,35255,1733206381424 2024-12-03T06:16:09,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, UNASSIGN because future has completed 2024-12-03T06:16:09,042 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T06:16:09,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424}] 2024-12-03T06:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T06:16:09,195 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(122): Close b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,195 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T06:16:09,195 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1722): Closing b3fa0bec8819326450abec32a5991191, disabling compactions & flushes 2024-12-03T06:16:09,195 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. after waiting 0 ms 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:09,196 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(2902): Flushing b3fa0bec8819326450abec32a5991191 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=A 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=B 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3fa0bec8819326450abec32a5991191, store=C 2024-12-03T06:16:09,196 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T06:16:09,209 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120305e439b386d14882932239f979e14f95_b3fa0bec8819326450abec32a5991191 is 50, key is test_row_0/A:col10/1733206569030/Put/seqid=0 2024-12-03T06:16:09,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742464_1640 (size=12454) 2024-12-03T06:16:09,216 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:16:09,220 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120305e439b386d14882932239f979e14f95_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120305e439b386d14882932239f979e14f95_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,222 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/512d4a5e2eea4e20ab4f609eceb3ab82, store: [table=TestAcidGuarantees family=A region=b3fa0bec8819326450abec32a5991191] 2024-12-03T06:16:09,222 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/512d4a5e2eea4e20ab4f609eceb3ab82 is 175, key is test_row_0/A:col10/1733206569030/Put/seqid=0 2024-12-03T06:16:09,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742465_1641 (size=31255) 2024-12-03T06:16:09,233 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/512d4a5e2eea4e20ab4f609eceb3ab82 2024-12-03T06:16:09,239 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9c926841acfc40d6bc285dc9e16a6b86 is 50, key is test_row_0/B:col10/1733206569030/Put/seqid=0 2024-12-03T06:16:09,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742466_1642 (size=12301) 2024-12-03T06:16:09,246 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9c926841acfc40d6bc285dc9e16a6b86 2024-12-03T06:16:09,254 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/25e48034fff643bdad1421bd25986603 is 50, key is test_row_0/C:col10/1733206569030/Put/seqid=0 2024-12-03T06:16:09,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742467_1643 (size=12301) 2024-12-03T06:16:09,263 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/25e48034fff643bdad1421bd25986603 2024-12-03T06:16:09,266 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/A/512d4a5e2eea4e20ab4f609eceb3ab82 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/512d4a5e2eea4e20ab4f609eceb3ab82 2024-12-03T06:16:09,270 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/512d4a5e2eea4e20ab4f609eceb3ab82, entries=150, sequenceid=285, filesize=30.5 K 2024-12-03T06:16:09,270 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/B/9c926841acfc40d6bc285dc9e16a6b86 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9c926841acfc40d6bc285dc9e16a6b86 2024-12-03T06:16:09,275 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9c926841acfc40d6bc285dc9e16a6b86, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T06:16:09,275 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/.tmp/C/25e48034fff643bdad1421bd25986603 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/25e48034fff643bdad1421bd25986603 2024-12-03T06:16:09,278 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/25e48034fff643bdad1421bd25986603, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T06:16:09,279 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b3fa0bec8819326450abec32a5991191 in 83ms, sequenceid=285, compaction requested=true 2024-12-03T06:16:09,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90] to archive 2024-12-03T06:16:09,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:16:09,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/07e09cab2c9d4c929bf7887480e3f5f1 2024-12-03T06:16:09,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/caafb878bab541d2954e3ea4e5562b96 2024-12-03T06:16:09,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/9ae42da0b8114e45a23ab415b03ebe15 2024-12-03T06:16:09,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/de29e94b125648d2bb73e998561144c6 2024-12-03T06:16:09,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/55b4bd0e16d74a3e9f8201f4addffc0b 2024-12-03T06:16:09,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/a4d98b21dc064114b137a95136802ad0 2024-12-03T06:16:09,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/06614d32d3b440b4b769db8cf6824990 2024-12-03T06:16:09,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/e1c78fc25ec84659b27275be7ba8d382 2024-12-03T06:16:09,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/834c9c192e894474acdb7733cf203d3a 2024-12-03T06:16:09,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/bdb2e742f063494494ecdd69364c1660 2024-12-03T06:16:09,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/49cb691e03704ab5af7d31317f51321a 2024-12-03T06:16:09,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/0775d3b2fb9a428fbee04d73456500ca 2024-12-03T06:16:09,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/cdafffb32ee84b879e01371aaaccf6ba 2024-12-03T06:16:09,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/b13ff4fc83d24a9ebc089cb854199463 2024-12-03T06:16:09,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/dc2bc39d39e34326976bfa082ef386f1 2024-12-03T06:16:09,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/945b35d907504ac7b00c54c02fd577b5 2024-12-03T06:16:09,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/32bdc8ce5e424f59ae8a154e53e8c35b 2024-12-03T06:16:09,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/5c214225c5fe4165978264086b447a90 2024-12-03T06:16:09,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c23e7464d0f34e1ca447d049f25ad47f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2ff5042d13e45b4a9b8da8c32462e03, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ee52b882659043909b2eeb2f7e804cbe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/10b8792f208943998f22199b9d550fb7, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a] to archive 2024-12-03T06:16:09,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:16:09,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/cb1ed8309dbb45c1b1774705aad33fe7 2024-12-03T06:16:09,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/4a1210c9a81c417b881c086cccb8612a 2024-12-03T06:16:09,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c23e7464d0f34e1ca447d049f25ad47f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c23e7464d0f34e1ca447d049f25ad47f 2024-12-03T06:16:09,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/d3c1f302e5384a86aa5c7ad6d007cdb3 2024-12-03T06:16:09,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/415771db5e634153a08fab3362e57227 2024-12-03T06:16:09,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2ff5042d13e45b4a9b8da8c32462e03 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2ff5042d13e45b4a9b8da8c32462e03 2024-12-03T06:16:09,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/5503d068dbbe45da8209ac86fea843f0 2024-12-03T06:16:09,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c77a0bc3313c4c06a5857a7c77db431e 2024-12-03T06:16:09,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ee52b882659043909b2eeb2f7e804cbe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ee52b882659043909b2eeb2f7e804cbe 2024-12-03T06:16:09,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/7aa390823e2a47a0970d89df3e1f8f2e 2024-12-03T06:16:09,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/c41158682ba44fadb34b6e17a342448c 2024-12-03T06:16:09,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/6c73ec0cb7244e1c81d3eaff68ccfdfe 2024-12-03T06:16:09,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/10b8792f208943998f22199b9d550fb7 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/10b8792f208943998f22199b9d550fb7 2024-12-03T06:16:09,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9b7d590ac04f4424bd2df48ed0f746e6 2024-12-03T06:16:09,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/b2308ed1f2fb49c0bb0f7a9cd9413d86 2024-12-03T06:16:09,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9e061b264b914eada7969f24da8441b8 2024-12-03T06:16:09,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/622e0e1776874a829944ec2c076fc9be 2024-12-03T06:16:09,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/24b0b1d3b1ba4fe58310968abbdaea3a 2024-12-03T06:16:09,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/789448c7c153409c99239caee331787a, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/71926db6685544fd919535f6bf576682, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a1230bc3eb63448892efb6b2139d73c8, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d0c00708331b4394b0f295f5be2632aa, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf] to archive 2024-12-03T06:16:09,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T06:16:09,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/3e81515d839045909ddecbf6de5540a0 2024-12-03T06:16:09,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7eb5217a598b4cd3ba0f83c93bd9e363 2024-12-03T06:16:09,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/789448c7c153409c99239caee331787a to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/789448c7c153409c99239caee331787a 2024-12-03T06:16:09,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/1f74ebd596454d649c4edccbbc35183f 2024-12-03T06:16:09,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9c965213b7734f34bc2ed5ab7c4e9374 2024-12-03T06:16:09,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/71926db6685544fd919535f6bf576682 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/71926db6685544fd919535f6bf576682 2024-12-03T06:16:09,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a608212238fb49caa6d866133a3887a4 2024-12-03T06:16:09,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/0ca78ed52d024d5dbcd8d9b1d79c3240 2024-12-03T06:16:09,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a1230bc3eb63448892efb6b2139d73c8 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/a1230bc3eb63448892efb6b2139d73c8 2024-12-03T06:16:09,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/87a8449b6fb8438e8509864287c81791 2024-12-03T06:16:09,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/171d9b41338a40b997ff8dbdfff52590 2024-12-03T06:16:09,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d28177d96f79444aac706b97c9a49e82 2024-12-03T06:16:09,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d0c00708331b4394b0f295f5be2632aa to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d0c00708331b4394b0f295f5be2632aa 2024-12-03T06:16:09,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/b2310174ffde4758904dccc2f8e0941c 2024-12-03T06:16:09,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/5710ba21756041cbb42f46ada3732abb 2024-12-03T06:16:09,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/9e1e391c9fe24bf8b7252b64388a3a65 2024-12-03T06:16:09,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/7b2e7313edcf4d43aa6d954f291ef9ed 2024-12-03T06:16:09,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/8c1615a4f00b449097eb8509a7ff76cf 2024-12-03T06:16:09,344 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits/288.seqid, newMaxSeqId=288, maxSeqId=4 2024-12-03T06:16:09,344 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. 2024-12-03T06:16:09,344 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1676): Region close journal for b3fa0bec8819326450abec32a5991191: Waiting for close lock at 1733206569195Running coprocessor pre-close hooks at 1733206569195Disabling compacts and flushes for region at 1733206569195Disabling writes for close at 1733206569196 (+1 ms)Obtaining lock to block concurrent updates at 1733206569196Preparing flush snapshotting stores in b3fa0bec8819326450abec32a5991191 at 1733206569196Finished memstore snapshotting TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733206569196Flushing stores of TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191. at 1733206569197 (+1 ms)Flushing b3fa0bec8819326450abec32a5991191/A: creating writer at 1733206569197Flushing mob file b3fa0bec8819326450abec32a5991191/A: appending metadata at 1733206569209 (+12 ms)Flushing mob file b3fa0bec8819326450abec32a5991191/A: closing flushed file at 1733206569209Flushing b3fa0bec8819326450abec32a5991191/A: appending metadata at 1733206569222 (+13 ms)Flushing b3fa0bec8819326450abec32a5991191/A: closing flushed file at 1733206569222Flushing b3fa0bec8819326450abec32a5991191/B: creating writer at 1733206569237 (+15 ms)Flushing b3fa0bec8819326450abec32a5991191/B: appending metadata at 1733206569239 (+2 ms)Flushing b3fa0bec8819326450abec32a5991191/B: closing flushed file at 1733206569239Flushing b3fa0bec8819326450abec32a5991191/C: creating writer at 1733206569250 (+11 ms)Flushing b3fa0bec8819326450abec32a5991191/C: appending metadata at 1733206569253 (+3 ms)Flushing b3fa0bec8819326450abec32a5991191/C: closing flushed file at 1733206569253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8e70c46: reopening flushed file at 1733206569266 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e7b4b20: reopening flushed file at 1733206569270 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10fd0ddd: reopening flushed file at 1733206569275 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b3fa0bec8819326450abec32a5991191 in 83ms, sequenceid=285, compaction requested=true at 1733206569279 (+4 ms)Writing region close event to WAL at 1733206569341 (+62 ms)Running coprocessor post-close hooks at 1733206569344 (+3 ms)Closed at 1733206569344 2024-12-03T06:16:09,346 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(157): Closed b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,347 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=b3fa0bec8819326450abec32a5991191, regionState=CLOSED 2024-12-03T06:16:09,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 because future has completed 2024-12-03T06:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T06:16:09,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-12-03T06:16:09,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseRegionProcedure b3fa0bec8819326450abec32a5991191, server=122cddc95ec8,35255,1733206381424 in 307 msec 2024-12-03T06:16:09,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-12-03T06:16:09,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3fa0bec8819326450abec32a5991191, UNASSIGN in 312 msec 2024-12-03T06:16:09,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-12-03T06:16:09,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 315 msec 2024-12-03T06:16:09,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206569355"}]},"ts":"1733206569355"} 2024-12-03T06:16:09,357 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T06:16:09,357 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T06:16:09,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 325 msec 2024-12-03T06:16:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T06:16:09,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:16:09,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T06:16:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] procedure2.ProcedureExecutor(1139): Stored pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,662 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-12-03T06:16:09,663 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=184, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,664 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C, FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits] 2024-12-03T06:16:09,668 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/1167148d4f6b4022bf3ffb7a0df5dcef to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/1167148d4f6b4022bf3ffb7a0df5dcef 2024-12-03T06:16:09,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/4161be093f70463aafe43549ff1fab4e to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/4161be093f70463aafe43549ff1fab4e 2024-12-03T06:16:09,670 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/512d4a5e2eea4e20ab4f609eceb3ab82 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/A/512d4a5e2eea4e20ab4f609eceb3ab82 2024-12-03T06:16:09,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/1a32b9c4af3d4c7bb3365a11ab8d2b13 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/1a32b9c4af3d4c7bb3365a11ab8d2b13 2024-12-03T06:16:09,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9c926841acfc40d6bc285dc9e16a6b86 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/9c926841acfc40d6bc285dc9e16a6b86 2024-12-03T06:16:09,673 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ad765e0b57074075a7f133b0bcae447f to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/B/ad765e0b57074075a7f133b0bcae447f 2024-12-03T06:16:09,675 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/25e48034fff643bdad1421bd25986603 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/25e48034fff643bdad1421bd25986603 2024-12-03T06:16:09,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/603eecd42c5a43dab555c466e9ca1d8d to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/603eecd42c5a43dab555c466e9ca1d8d 2024-12-03T06:16:09,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d10dc4903781473ca8154a23a4729be9 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/C/d10dc4903781473ca8154a23a4729be9 2024-12-03T06:16:09,680 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits/288.seqid to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191/recovered.edits/288.seqid 2024-12-03T06:16:09,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/default/TestAcidGuarantees/b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T06:16:09,681 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:16:09,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T06:16:09,684 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120305e439b386d14882932239f979e14f95_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120305e439b386d14882932239f979e14f95_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c7eaab600134f778cf5a2a927e7080f_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c7eaab600134f778cf5a2a927e7080f_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,686 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033bbb975cfdbf4d74ba4bfebe2397ce6b_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033bbb975cfdbf4d74ba4bfebe2397ce6b_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033efbe5801f4a44e99d34a1e377c95ca0_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033efbe5801f4a44e99d34a1e377c95ca0_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fd1933ac88b4e0b99824a5d8c24f543_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fd1933ac88b4e0b99824a5d8c24f543_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d5c13bea08d4d4db68867878a644fbf_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d5c13bea08d4d4db68867878a644fbf_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,689 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037b8cb5e9d16649f8af9209e422f43b4c_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037b8cb5e9d16649f8af9209e422f43b4c_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,690 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203859e5f0825e94149857c7c7ecc53bd12_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203859e5f0825e94149857c7c7ecc53bd12_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a6a6c907fe8549ca9c28cc1c99f4db17_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a6a6c907fe8549ca9c28cc1c99f4db17_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2da6f6c460b49029ffc7133e2f3ec7e_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2da6f6c460b49029ffc7133e2f3ec7e_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,692 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2148d48901c4f198787140c3af7fe08_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2148d48901c4f198787140c3af7fe08_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,693 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d29618e7212f4b2ea9ac8cef7a0613c6_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d29618e7212f4b2ea9ac8cef7a0613c6_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,694 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e67539f945194faaa5eea0c5378d8b5c_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e67539f945194faaa5eea0c5378d8b5c_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e757a00eaf474690ae21d6619b57acc7_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e757a00eaf474690ae21d6619b57acc7_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1148dc619b84938882064bec54e5260_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1148dc619b84938882064bec54e5260_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f44d5fb8dcc4468e9c3139933b98b08a_b3fa0bec8819326450abec32a5991191 to hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f44d5fb8dcc4468e9c3139933b98b08a_b3fa0bec8819326450abec32a5991191 2024-12-03T06:16:09,699 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T06:16:09,701 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=184, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,703 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T06:16:09,705 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T06:16:09,706 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=184, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,706 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T06:16:09,706 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733206569706"}]},"ts":"9223372036854775807"} 2024-12-03T06:16:09,712 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T06:16:09,712 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b3fa0bec8819326450abec32a5991191, NAME => 'TestAcidGuarantees,,1733206538457.b3fa0bec8819326450abec32a5991191.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T06:16:09,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T06:16:09,713 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733206569713"}]},"ts":"9223372036854775807"} 2024-12-03T06:16:09,715 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T06:16:09,716 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=184, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T06:16:09,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-03T06:16:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-12-03T06:16:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T06:16:09,770 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T06:16:09,780 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=415 (was 411) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11670 (was 11424) - AvailableMemoryMB LEAK? - 2024-12-03T06:16:09,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T06:16:09,780 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:16:09,780 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:16:09,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:16:09,780 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T06:16:09,780 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-12-03T06:16:09,781 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=122cddc95ec8,42621,1733206380709 2024-12-03T06:16:09,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T06:16:09,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T06:16:09,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:16:09,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:16:09,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:16:09,815 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:16:09,815 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:16:09,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,816 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '122cddc95ec8,35255,1733206381424' ***** 2024-12-03T06:16:09,816 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T06:16:09,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:16:09,816 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T06:16:09,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:16:09,816 INFO [RS:0;122cddc95ec8:35255 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T06:16:09,816 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T06:16:09,816 INFO [RS:0;122cddc95ec8:35255 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(959): stopping server 122cddc95ec8,35255,1733206381424 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;122cddc95ec8:35255. 2024-12-03T06:16:09,817 DEBUG [RS:0;122cddc95ec8:35255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:16:09,817 DEBUG [RS:0;122cddc95ec8:35255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T06:16:09,817 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T06:16:09,819 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T06:16:09,819 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T06:16:09,820 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:09,820 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T06:16:09,820 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T06:16:09,820 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T06:16:09,820 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T06:16:09,820 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T06:16:09,820 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T06:16:09,847 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/info/3724a168e9004113aeb17d4eaa6e32b0 is 95, key is TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6./info:/1733206413698/DeleteFamily/seqid=0 2024-12-03T06:16:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742468_1644 (size=6514) 2024-12-03T06:16:09,911 INFO [regionserver/122cddc95ec8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:16:10,020 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:10,220 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:10,251 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/info/3724a168e9004113aeb17d4eaa6e32b0 2024-12-03T06:16:10,268 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/ns/d60777355c09440b8d56fde7bebb908e is 93, key is TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6./ns:/1733206413649/DeleteFamily/seqid=0 2024-12-03T06:16:10,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742469_1645 (size=5906) 2024-12-03T06:16:10,420 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:10,474 INFO [regionserver/122cddc95ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T06:16:10,475 INFO [regionserver/122cddc95ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T06:16:10,620 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:10,672 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/ns/d60777355c09440b8d56fde7bebb908e 2024-12-03T06:16:10,693 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/rep_barrier/16819cd126374bfc88f4f2f41e8222d9 is 102, key is TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6./rep_barrier:/1733206413649/DeleteFamily/seqid=0 2024-12-03T06:16:10,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742470_1646 (size=6025) 2024-12-03T06:16:10,821 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:11,021 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:11,047 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T06:16:11,098 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/rep_barrier/16819cd126374bfc88f4f2f41e8222d9 2024-12-03T06:16:11,116 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/table/b4690ec15a1b4c63bcacfd7fedc45971 is 96, key is TestAcidGuarantees,,1733206385744.8ec9e85614363b6e4e059ef6575a1df6./table:/1733206413649/DeleteFamily/seqid=0 2024-12-03T06:16:11,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742471_1647 (size=5936) 2024-12-03T06:16:11,221 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:11,421 DEBUG [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T06:16:11,520 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/table/b4690ec15a1b4c63bcacfd7fedc45971 2024-12-03T06:16:11,523 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/info/3724a168e9004113aeb17d4eaa6e32b0 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/info/3724a168e9004113aeb17d4eaa6e32b0 2024-12-03T06:16:11,525 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/info/3724a168e9004113aeb17d4eaa6e32b0, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T06:16:11,525 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/ns/d60777355c09440b8d56fde7bebb908e as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/ns/d60777355c09440b8d56fde7bebb908e 2024-12-03T06:16:11,527 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/ns/d60777355c09440b8d56fde7bebb908e, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T06:16:11,528 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/rep_barrier/16819cd126374bfc88f4f2f41e8222d9 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/rep_barrier/16819cd126374bfc88f4f2f41e8222d9 2024-12-03T06:16:11,530 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/rep_barrier/16819cd126374bfc88f4f2f41e8222d9, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T06:16:11,530 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/.tmp/table/b4690ec15a1b4c63bcacfd7fedc45971 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/table/b4690ec15a1b4c63bcacfd7fedc45971 2024-12-03T06:16:11,533 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/table/b4690ec15a1b4c63bcacfd7fedc45971, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T06:16:11,534 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1714ms, sequenceid=90, compaction requested=false 2024-12-03T06:16:11,540 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T06:16:11,541 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T06:16:11,541 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T06:16:11,541 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733206569819Running coprocessor pre-close hooks at 1733206569820 (+1 ms)Disabling compacts and flushes for region at 1733206569820Disabling writes for close at 1733206569820Obtaining lock to block concurrent updates at 1733206569820Preparing flush snapshotting stores in 1588230740 at 1733206569820Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733206569821 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733206569821Flushing 1588230740/info: creating writer at 1733206569821Flushing 1588230740/info: appending metadata at 1733206569846 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733206569846Flushing 1588230740/ns: creating writer at 1733206570255 (+409 ms)Flushing 1588230740/ns: appending metadata at 1733206570268 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733206570268Flushing 1588230740/rep_barrier: creating writer at 1733206570675 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733206570693 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733206570693Flushing 1588230740/table: creating writer at 1733206571101 (+408 ms)Flushing 1588230740/table: appending metadata at 1733206571115 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733206571115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41bfed89: reopening flushed file at 1733206571522 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2253ad6d: reopening flushed file at 1733206571525 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@376bf85e: reopening flushed file at 1733206571528 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68bd997d: reopening flushed file at 1733206571530 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1714ms, sequenceid=90, compaction requested=false at 1733206571534 (+4 ms)Writing region close event to WAL at 1733206571538 (+4 ms)Running coprocessor post-close hooks at 1733206571540 (+2 ms)Closed at 1733206571541 (+1 ms) 2024-12-03T06:16:11,541 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T06:16:11,622 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(976): stopping server 122cddc95ec8,35255,1733206381424; all regions closed. 2024-12-03T06:16:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741834_1010 (size=25344) 2024-12-03T06:16:11,627 DEBUG [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/oldWALs 2024-12-03T06:16:11,628 INFO [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 122cddc95ec8%2C35255%2C1733206381424.meta:.meta(num 1733206385264) 2024-12-03T06:16:11,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741832_1008 (size=17167198) 2024-12-03T06:16:11,632 DEBUG [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/oldWALs 2024-12-03T06:16:11,632 INFO [RS:0;122cddc95ec8:35255 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 122cddc95ec8%2C35255%2C1733206381424:(num 1733206384634) 2024-12-03T06:16:11,632 DEBUG [RS:0;122cddc95ec8:35255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:16:11,632 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:16:11,632 INFO [RS:0;122cddc95ec8:35255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T06:16:11,632 INFO [RS:0;122cddc95ec8:35255 {}] hbase.ChoreService(370): Chore service for: regionserver/122cddc95ec8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T06:16:11,633 INFO [RS:0;122cddc95ec8:35255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T06:16:11,633 INFO [regionserver/122cddc95ec8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T06:16:11,634 INFO [RS:0;122cddc95ec8:35255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35255 2024-12-03T06:16:11,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T06:16:11,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/122cddc95ec8,35255,1733206381424 2024-12-03T06:16:11,673 INFO [RS:0;122cddc95ec8:35255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:16:11,681 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [122cddc95ec8,35255,1733206381424] 2024-12-03T06:16:11,689 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/122cddc95ec8,35255,1733206381424 already deleted, retry=false 2024-12-03T06:16:11,690 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 122cddc95ec8,35255,1733206381424 expired; onlineServers=0 2024-12-03T06:16:11,690 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '122cddc95ec8,42621,1733206380709' ***** 2024-12-03T06:16:11,690 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T06:16:11,690 INFO [M:0;122cddc95ec8:42621 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:16:11,690 INFO [M:0;122cddc95ec8:42621 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T06:16:11,690 DEBUG [M:0;122cddc95ec8:42621 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T06:16:11,690 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T06:16:11,690 DEBUG [M:0;122cddc95ec8:42621 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T06:16:11,690 DEBUG [master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206384350 {}] cleaner.HFileCleaner(306): Exit Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206384350,5,FailOnTimeoutGroup] 2024-12-03T06:16:11,690 DEBUG [master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206384341 {}] cleaner.HFileCleaner(306): Exit Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206384341,5,FailOnTimeoutGroup] 2024-12-03T06:16:11,691 INFO [M:0;122cddc95ec8:42621 {}] hbase.ChoreService(370): Chore service for: master/122cddc95ec8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T06:16:11,691 INFO [M:0;122cddc95ec8:42621 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T06:16:11,691 DEBUG [M:0;122cddc95ec8:42621 {}] master.HMaster(1795): Stopping service threads 2024-12-03T06:16:11,691 INFO [M:0;122cddc95ec8:42621 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T06:16:11,691 INFO [M:0;122cddc95ec8:42621 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:16:11,691 ERROR [M:0;122cddc95ec8:42621 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T06:16:11,692 INFO [M:0;122cddc95ec8:42621 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:16:11,692 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T06:16:11,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T06:16:11,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:16:11,698 DEBUG [M:0;122cddc95ec8:42621 {}] zookeeper.ZKUtil(347): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T06:16:11,698 WARN [M:0;122cddc95ec8:42621 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T06:16:11,699 INFO [M:0;122cddc95ec8:42621 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/.lastflushedseqids 2024-12-03T06:16:11,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742472_1648 (size=72) 2024-12-03T06:16:11,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:16:11,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35255-0x1009ea88bf20001, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:16:11,782 INFO [RS:0;122cddc95ec8:35255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:16:11,783 INFO [RS:0;122cddc95ec8:35255 {}] regionserver.HRegionServer(1031): Exiting; stopping=122cddc95ec8,35255,1733206381424; zookeeper connection closed. 2024-12-03T06:16:11,783 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63bb6fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63bb6fa 2024-12-03T06:16:11,783 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T06:16:12,115 INFO [M:0;122cddc95ec8:42621 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:16:12,115 INFO [M:0;122cddc95ec8:42621 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T06:16:12,115 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T06:16:12,115 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:16:12,115 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:16:12,115 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T06:16:12,115 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:16:12,115 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=825.86 KB heapSize=1015.76 KB 2024-12-03T06:16:12,131 DEBUG [M:0;122cddc95ec8:42621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f26727bf4e624e3cb44a81242f5ec828 is 82, key is hbase:meta,,1/info:regioninfo/1733206385346/Put/seqid=0 2024-12-03T06:16:12,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742473_1649 (size=5672) 2024-12-03T06:16:12,535 INFO [M:0;122cddc95ec8:42621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2350 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f26727bf4e624e3cb44a81242f5ec828 2024-12-03T06:16:12,555 DEBUG [M:0;122cddc95ec8:42621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e078fa096216496e970ba4a7b3a244cb is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9F/proc:d/1733206541572/Put/seqid=0 2024-12-03T06:16:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742474_1650 (size=47793) 2024-12-03T06:16:12,969 INFO [M:0;122cddc95ec8:42621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=825.31 KB at sequenceid=2350 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e078fa096216496e970ba4a7b3a244cb 2024-12-03T06:16:12,972 INFO [M:0;122cddc95ec8:42621 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e078fa096216496e970ba4a7b3a244cb 2024-12-03T06:16:12,985 DEBUG [M:0;122cddc95ec8:42621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa2866f78960483a965dc42063737f54 is 69, key is 122cddc95ec8,35255,1733206381424/rs:state/1733206384367/Put/seqid=0 2024-12-03T06:16:12,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073742475_1651 (size=5156) 2024-12-03T06:16:13,399 INFO [M:0;122cddc95ec8:42621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2350 (bloomFilter=true), to=hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa2866f78960483a965dc42063737f54 2024-12-03T06:16:13,404 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f26727bf4e624e3cb44a81242f5ec828 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f26727bf4e624e3cb44a81242f5ec828 2024-12-03T06:16:13,407 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f26727bf4e624e3cb44a81242f5ec828, entries=8, sequenceid=2350, filesize=5.5 K 2024-12-03T06:16:13,407 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e078fa096216496e970ba4a7b3a244cb as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e078fa096216496e970ba4a7b3a244cb 2024-12-03T06:16:13,410 INFO [M:0;122cddc95ec8:42621 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e078fa096216496e970ba4a7b3a244cb 2024-12-03T06:16:13,410 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e078fa096216496e970ba4a7b3a244cb, entries=184, sequenceid=2350, filesize=46.7 K 2024-12-03T06:16:13,411 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa2866f78960483a965dc42063737f54 as hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa2866f78960483a965dc42063737f54 2024-12-03T06:16:13,414 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44351/user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa2866f78960483a965dc42063737f54, entries=1, sequenceid=2350, filesize=5.0 K 2024-12-03T06:16:13,414 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(3140): Finished flush of dataSize ~825.86 KB/845683, heapSize ~1015.46 KB/1039832, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2350, compaction requested=false 2024-12-03T06:16:13,416 INFO [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:16:13,416 DEBUG [M:0;122cddc95ec8:42621 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733206572115Disabling compacts and flushes for region at 1733206572115Disabling writes for close at 1733206572115Obtaining lock to block concurrent updates at 1733206572115Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733206572115Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=845683, getHeapSize=1040072, getOffHeapSize=0, getCellsCount=2559 at 1733206572115Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733206572116 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733206572116Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733206572131 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733206572131Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733206572538 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733206572555 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733206572555Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733206572972 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733206572985 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733206572985Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4db343: reopening flushed file at 1733206573403 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4013c018: reopening flushed file at 1733206573407 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61f10277: reopening flushed file at 1733206573410 (+3 ms)Finished flush of dataSize ~825.86 KB/845683, heapSize ~1015.46 KB/1039832, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2350, compaction requested=false at 1733206573414 (+4 ms)Writing region close event to WAL at 1733206573416 (+2 ms)Closed at 1733206573416 2024-12-03T06:16:13,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45259 is added to blk_1073741830_1006 (size=999688) 2024-12-03T06:16:13,420 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6dac923a-9d10-35c6-c5bd-25393023135c/MasterData/WALs/122cddc95ec8,42621,1733206380709/122cddc95ec8%2C42621%2C1733206380709.1733206383562 not finished, retry = 0 2024-12-03T06:16:13,521 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T06:16:13,521 INFO [M:0;122cddc95ec8:42621 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T06:16:13,521 INFO [M:0;122cddc95ec8:42621 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42621 2024-12-03T06:16:13,521 INFO [M:0;122cddc95ec8:42621 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:16:13,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:16:13,636 INFO [M:0;122cddc95ec8:42621 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:16:13,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42621-0x1009ea88bf20000, quorum=127.0.0.1:53118, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:16:13,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:16:13,642 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:16:13,642 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:16:13,643 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:16:13,643 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.log.dir/,STOPPED} 2024-12-03T06:16:13,645 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T06:16:13,645 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T06:16:13,645 WARN [BP-1574928969-172.17.0.2-1733206377249 heartbeating to localhost/127.0.0.1:44351 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T06:16:13,645 WARN [BP-1574928969-172.17.0.2-1733206377249 heartbeating to localhost/127.0.0.1:44351 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1574928969-172.17.0.2-1733206377249 (Datanode Uuid 5db34709-8f31-43be-8f27-fa9b2176541b) service to localhost/127.0.0.1:44351 2024-12-03T06:16:13,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data1/current/BP-1574928969-172.17.0.2-1733206377249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:16:13,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/cluster_5893b093-5c9a-073a-086a-74d036aebb38/data/data2/current/BP-1574928969-172.17.0.2-1733206377249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:16:13,649 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T06:16:13,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T06:16:13,657 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:16:13,657 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:16:13,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:16:13,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ab7da33-de31-26c0-4ca1-0be7c4630998/hadoop.log.dir/,STOPPED} 2024-12-03T06:16:13,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T06:16:13,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down